From 7a62de0ad1ad5fff105f7913b654ebd139ed543c Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 3 Apr 2023 14:56:22 +0700 Subject: [PATCH 001/112] draft --- mm2src/coins/lp_coins.rs | 2 ++ mm2src/coins/nft_storage/mod.rs | 2 ++ mm2src/coins/nft_storage/sql_storage.rs | 1 + mm2src/coins/nft_storage/wasm_storage.rs | 1 + 4 files changed, 6 insertions(+) create mode 100644 mm2src/coins/nft_storage/mod.rs create mode 100644 mm2src/coins/nft_storage/sql_storage.rs create mode 100644 mm2src/coins/nft_storage/wasm_storage.rs diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 78118f54b7..423a90a519 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -285,7 +285,9 @@ use utxo::{BlockchainNetwork, GenerateTxError, UtxoFeeDetails, UtxoTx}; #[cfg(feature = "enable-nft-integration")] use nft::nft_errors::GetNftInfoError; +#[cfg(feature = "enable-nft-integration")] pub mod nft_storage; #[cfg(not(target_arch = "wasm32"))] pub mod z_coin; + #[cfg(not(target_arch = "wasm32"))] use z_coin::ZCoin; pub type TransactionFut = Box + Send>; diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs new file mode 100644 index 0000000000..80af3a2fe3 --- /dev/null +++ b/mm2src/coins/nft_storage/mod.rs @@ -0,0 +1,2 @@ +#[cfg(not(target_arch = "wasm32"))] mod sql_storage; +#[cfg(target_arch = "wasm32")] mod wasm_storage; diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -0,0 +1 @@ + diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -0,0 +1 @@ + From 350f1d576abc09c88d9d130eaab1e69bec75a222 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 5 Apr 2023 11:47:49 +0700 Subject: [PATCH 002/112] wip --- mm2src/coins/lp_coins.rs | 8 ++++++++ mm2src/coins/nft_storage/mod.rs | 4 ++-- mm2src/coins/nft_storage/wasm_storage.rs | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 423a90a519..c23254e981 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -98,6 +98,8 @@ cfg_wasm32! { use mm2_db::indexed_db::{ConstructibleDb, DbLocked, SharedDb}; use tx_history_storage::wasm::{clear_tx_history, load_tx_history, save_tx_history, TxHistoryDb}; pub type TxHistoryDbLocked<'a> = DbLocked<'a, TxHistoryDb>; + #[cfg(feature = "enable-nft-integration")] + use nft_storage::wasm_storage::NftCacheDb; } // using custom copy of try_fus as futures crate was renamed to futures01 @@ -2450,6 +2452,9 @@ pub struct CoinsContext { tx_history_db: SharedDb, #[cfg(target_arch = "wasm32")] hd_wallet_db: SharedDb, + #[cfg(feature = "enable-nft-integration")] + #[cfg(target_arch = "wasm32")] + pub nft_cache_db: SharedDb, } #[derive(Debug)] @@ -2474,6 +2479,9 @@ impl CoinsContext { tx_history_db: ConstructibleDb::new(ctx).into_shared(), #[cfg(target_arch = "wasm32")] hd_wallet_db: ConstructibleDb::new_shared_db(ctx).into_shared(), + #[cfg(feature = "enable-nft-integration")] + #[cfg(target_arch = "wasm32")] + nft_cache_db: ConstructibleDb::new(ctx).into_shared(), }) }))) } diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 80af3a2fe3..c9bbbf79c1 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,2 +1,2 @@ -#[cfg(not(target_arch = "wasm32"))] mod sql_storage; -#[cfg(target_arch = "wasm32")] mod wasm_storage; +#[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; +#[cfg(target_arch = "wasm32")] pub mod wasm_storage; diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index 8b13789179..e2e9f83577 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1 +1,25 @@ +use async_trait::async_trait; +pub use mm2_db::indexed_db::InitDbResult; +use mm2_db::indexed_db::{DbIdentifier, DbInstance, IndexedDb, IndexedDbBuilder}; +const DB_NAME: &str = "nft_cache"; +const DB_VERSION: u32 = 1; + +pub struct NftCacheDb { + inner: IndexedDb, +} + +#[async_trait] +impl DbInstance for NftCacheDb { + fn db_name() -> &'static str { DB_NAME } + + async fn init(db_id: DbIdentifier) -> InitDbResult { + // todo add tables for each chain? + let inner = IndexedDbBuilder::new(db_id).with_version(DB_VERSION).build().await?; + Ok(NftCacheDb { inner }) + } +} + +impl NftCacheDb { + fn get_inner(&self) -> &IndexedDb { &self.inner } +} From 08ba26c330405b8b88b5d4745b4348fcefa9f719 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 6 Apr 2023 19:24:02 +0700 Subject: [PATCH 003/112] wip --- mm2src/coins/nft/nft_structs.rs | 4 +-- mm2src/coins/nft_storage/mod.rs | 44 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 09e247967c..70e39c780c 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -18,7 +18,7 @@ pub struct NftMetadataReq { #[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "UPPERCASE")] -pub(crate) enum Chain { +pub enum Chain { Avalanche, Bsc, Eth, @@ -209,7 +209,7 @@ pub struct NftTransfersReq { } #[derive(Debug, Serialize)] -pub(crate) struct NftTransferHistory { +pub struct NftTransferHistory { pub(crate) chain: Chain, pub(crate) block_number: u64, pub(crate) block_timestamp: String, diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index c9bbbf79c1..8d256687c0 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,2 +1,46 @@ +use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use async_trait::async_trait; +use mm2_err_handle::mm_error::NotMmError; +use mm2_err_handle::prelude::MmResult; + #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; + +pub trait NftListStorageError: std::fmt::Debug + NotMmError + Send {} +pub trait NftTxHistoryStorageError: std::fmt::Debug + NotMmError + Send {} + +#[async_trait] +pub trait NftListStorageOps { + type Error: NftListStorageError; + + /// Initializes tables in storage for the specified chain type. + async fn init(&self, chain: Chain) -> MmResult<(), Self::Error>; + + /// Whether tables are initialized for the specified chain. + async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error>; + + async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error>; + + async fn add_nfts(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send; +} + +#[async_trait] +pub trait NftTxHistoryStorageOps { + type Error: NftTxHistoryStorageError; + + /// Initializes tables in storage for the specified chain type. + async fn init(&self, chain: Chain) -> MmResult<(), Self::Error>; + + /// Whether tables are initialized for the specified chain. + async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error>; + + async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error>; + + async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send; +} From 7d0edf1be4ea23af403c8cc833be8a66c9d642d9 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 10 Apr 2023 15:39:44 +0700 Subject: [PATCH 004/112] wip --- mm2src/coins/nft_storage/mod.rs | 43 +++++++++++++++++++++++- mm2src/coins/nft_storage/sql_storage.rs | 22 ++++++++++++ mm2src/coins/nft_storage/wasm_storage.rs | 4 ++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 8d256687c0..9c39c7d59c 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,7 +1,9 @@ use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; use async_trait::async_trait; +use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::NotMmError; use mm2_err_handle::prelude::MmResult; +use std::marker::PhantomData; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; @@ -21,7 +23,7 @@ pub trait NftListStorageOps { async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error>; - async fn add_nfts(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; @@ -44,3 +46,42 @@ pub trait NftTxHistoryStorageOps { I: IntoIterator + Send + 'static, I::IntoIter: Send; } + +pub enum CreateNftStorageError { + Internal(String), +} + +pub trait StorageFactory { + fn create(ctx: &MmArc) -> MmResult + where + Self: Sized; +} + +/// `NftStorageBuilder` is used to create an instance that implements the `NftListStorageOps` +/// and `NftTxHistoryStorageOps` traits. +#[allow(dead_code)] +pub struct NftStorageBuilder<'a, T> { + ctx: &'a MmArc, + _phantom: PhantomData, +} + +impl<'a, T> NftStorageBuilder<'a, T> +where + T: NftListStorageOps + NftTxHistoryStorageOps + StorageFactory, +{ + #[inline] + pub fn new(ctx: &MmArc) -> NftStorageBuilder<'_, T> { + NftStorageBuilder { + ctx, + _phantom: Default::default(), + } + } + + #[inline] + pub fn build(self) -> MmResult + where + Self: Sized, + { + T::create(self.ctx) + } +} diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 8b13789179..a6477aa289 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1 +1,23 @@ +use crate::nft_storage::{CreateNftStorageError, StorageFactory}; +use db_common::sqlite::rusqlite::Connection; +use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::mm_error::{MmError, MmResult}; +use std::sync::{Arc, Mutex}; +#[derive(Clone)] +pub struct SqliteNftStorage(Arc>); + +impl SqliteNftStorage { + pub fn new(ctx: &MmArc) -> MmResult { + let sqlite_connection = ctx + .sqlite_connection + .ok_or(MmError::new(CreateNftStorageError::Internal( + "sqlite_connection is not initialized".to_owned(), + )))?; + Ok(SqliteNftStorage(sqlite_connection.clone())) + } +} + +impl StorageFactory for SqliteNftStorage { + fn create(ctx: &MmArc) -> MmResult { SqliteNftStorage::new(ctx) } +} diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index e2e9f83577..950dbf0528 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,10 +1,12 @@ use async_trait::async_trait; pub use mm2_db::indexed_db::InitDbResult; -use mm2_db::indexed_db::{DbIdentifier, DbInstance, IndexedDb, IndexedDbBuilder}; +use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder}; const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; +pub type NftCacheDBLocked<'a> = DbLocked<'a, NftCacheDb>; + pub struct NftCacheDb { inner: IndexedDb, } From 5df609efa6eaf5d7e1e97d9c7343a5c872c05b44 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 10 Apr 2023 15:50:15 +0700 Subject: [PATCH 005/112] remove StorageFactory --- mm2src/coins/nft_storage/mod.rs | 31 ++++++------------------- mm2src/coins/nft_storage/sql_storage.rs | 6 +---- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 9c39c7d59c..af6e4ab6cb 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,9 +1,9 @@ use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use crate::nft_storage::sql_storage::SqliteNftStorage; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::NotMmError; use mm2_err_handle::prelude::MmResult; -use std::marker::PhantomData; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; @@ -51,37 +51,20 @@ pub enum CreateNftStorageError { Internal(String), } -pub trait StorageFactory { - fn create(ctx: &MmArc) -> MmResult - where - Self: Sized; -} - /// `NftStorageBuilder` is used to create an instance that implements the `NftListStorageOps` /// and `NftTxHistoryStorageOps` traits. #[allow(dead_code)] -pub struct NftStorageBuilder<'a, T> { +pub struct NftStorageBuilder<'a> { ctx: &'a MmArc, - _phantom: PhantomData, } -impl<'a, T> NftStorageBuilder<'a, T> -where - T: NftListStorageOps + NftTxHistoryStorageOps + StorageFactory, -{ +impl<'a> NftStorageBuilder<'a> { #[inline] - pub fn new(ctx: &MmArc) -> NftStorageBuilder<'_, T> { - NftStorageBuilder { - ctx, - _phantom: Default::default(), - } - } + pub fn new(ctx: &MmArc) -> NftStorageBuilder<'_> { NftStorageBuilder { ctx } } #[inline] - pub fn build(self) -> MmResult - where - Self: Sized, - { - T::create(self.ctx) + pub fn build(self) -> MmResult { + #[cfg(not(target_arch = "wasm32"))] + SqliteNftStorage::new(self.ctx) } } diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index a6477aa289..fc5c3f2377 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,4 +1,4 @@ -use crate::nft_storage::{CreateNftStorageError, StorageFactory}; +use crate::nft_storage::CreateNftStorageError; use db_common::sqlite::rusqlite::Connection; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; @@ -17,7 +17,3 @@ impl SqliteNftStorage { Ok(SqliteNftStorage(sqlite_connection.clone())) } } - -impl StorageFactory for SqliteNftStorage { - fn create(ctx: &MmArc) -> MmResult { SqliteNftStorage::new(ctx) } -} From b0d965f62e9fdd25e9bb9fd8e8bcaa2ee9875c63 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 10 Apr 2023 16:14:42 +0700 Subject: [PATCH 006/112] add wasm target in NftStorageBuilder::build --- mm2src/coins/nft_storage/mod.rs | 6 +++--- mm2src/coins/nft_storage/wasm_storage.rs | 24 +++++++++++++++++++++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index af6e4ab6cb..257fbdd2fb 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,5 +1,4 @@ use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; -use crate::nft_storage::sql_storage::SqliteNftStorage; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::NotMmError; @@ -53,7 +52,6 @@ pub enum CreateNftStorageError { /// `NftStorageBuilder` is used to create an instance that implements the `NftListStorageOps` /// and `NftTxHistoryStorageOps` traits. -#[allow(dead_code)] pub struct NftStorageBuilder<'a> { ctx: &'a MmArc, } @@ -64,7 +62,9 @@ impl<'a> NftStorageBuilder<'a> { #[inline] pub fn build(self) -> MmResult { + #[cfg(target_arch = "wasm32")] + return wasm_storage::IndexedDbNftStorage::new(self.ctx); #[cfg(not(target_arch = "wasm32"))] - SqliteNftStorage::new(self.ctx) + sql_storage::SqliteNftStorage::new(self.ctx) } } diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index 950dbf0528..5aa603ebd8 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,6 +1,11 @@ +use crate::nft_storage::CreateNftStorageError; +use crate::CoinsContext; use async_trait::async_trait; +use mm2_core::mm_ctx::MmArc; pub use mm2_db::indexed_db::InitDbResult; -use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder}; +use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder, SharedDb}; +use mm2_err_handle::map_to_mm::MapToMmResult; +use mm2_err_handle::prelude::MmResult; const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; @@ -25,3 +30,20 @@ impl DbInstance for NftCacheDb { impl NftCacheDb { fn get_inner(&self) -> &IndexedDb { &self.inner } } + +#[derive(Clone)] +pub struct IndexedDbNftStorage { + db: SharedDb, +} + +impl IndexedDbNftStorage { + pub fn new(ctx: &MmArc) -> MmResult + where + Self: Sized, + { + let coins_ctx = CoinsContext::from_ctx(ctx).map_to_mm(CreateNftStorageError::Internal)?; + Ok(IndexedDbNftStorage { + db: coins_ctx.nft_cache_db.clone(), + }) + } +} From 6d840383191553e407a527afbc51e302369364fc Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 10 Apr 2023 19:09:55 +0700 Subject: [PATCH 007/112] add StorageOps todos --- mm2src/coins/nft_storage/sql_storage.rs | 42 +++++++++++++++++++++- mm2src/coins/nft_storage/wasm_storage.rs | 46 +++++++++++++++++++++--- 2 files changed, 82 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index fc5c3f2377..715de4fbc0 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,4 +1,6 @@ -use crate::nft_storage::CreateNftStorageError; +use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTxHistoryStorageOps}; +use async_trait::async_trait; use db_common::sqlite::rusqlite::Connection; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; @@ -17,3 +19,41 @@ impl SqliteNftStorage { Ok(SqliteNftStorage(sqlite_connection.clone())) } } + +#[async_trait] +impl NftListStorageOps for SqliteNftStorage { + type Error = (); + + async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + { + todo!() + } +} + +#[async_trait] +impl NftTxHistoryStorageOps for SqliteNftStorage { + type Error = (); + + async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + { + todo!() + } +} diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index 5aa603ebd8..b1d42a4910 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,4 +1,5 @@ -use crate::nft_storage::CreateNftStorageError; +use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTxHistoryStorageOps}; use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; @@ -37,13 +38,48 @@ pub struct IndexedDbNftStorage { } impl IndexedDbNftStorage { - pub fn new(ctx: &MmArc) -> MmResult - where - Self: Sized, - { + pub fn new(ctx: &MmArc) -> MmResult { let coins_ctx = CoinsContext::from_ctx(ctx).map_to_mm(CreateNftStorageError::Internal)?; Ok(IndexedDbNftStorage { db: coins_ctx.nft_cache_db.clone(), }) } } + +#[async_trait] +impl NftListStorageOps for IndexedDbNftStorage { + type Error = (); + + async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + { + todo!() + } +} + +#[async_trait] +impl NftTxHistoryStorageOps for IndexedDbNftStorage { + type Error = (); + + async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + + async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + { + todo!() + } +} From 9ef02b7b8ef857f66cd4b0d0b845770d806c153f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 11 Apr 2023 17:00:53 +0700 Subject: [PATCH 008/112] impl trait err for target err --- mm2src/coins/nft_storage/mod.rs | 4 ++-- mm2src/coins/nft_storage/sql_storage.rs | 28 ++++++++++++---------- mm2src/coins/nft_storage/wasm_storage.rs | 30 +++++++++++++++--------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 257fbdd2fb..ac52aa0fd1 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -18,7 +18,7 @@ pub trait NftListStorageOps { async fn init(&self, chain: Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. - async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error>; + async fn is_initialized_for(&self, chain: Chain) -> MmResult; async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error>; @@ -36,7 +36,7 @@ pub trait NftTxHistoryStorageOps { async fn init(&self, chain: Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. - async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error>; + async fn is_initialized_for(&self, chain: Chain) -> MmResult; async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error>; diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 715de4fbc0..b385644861 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,11 +1,15 @@ use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTxHistoryStorageOps}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, + NftTxHistoryStorageOps}; use async_trait::async_trait; -use db_common::sqlite::rusqlite::Connection; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use std::sync::{Arc, Mutex}; +impl NftListStorageError for SqlError {} +impl NftTxHistoryStorageError for SqlError {} + #[derive(Clone)] pub struct SqliteNftStorage(Arc>); @@ -22,15 +26,15 @@ impl SqliteNftStorage { #[async_trait] impl NftListStorageOps for SqliteNftStorage { - type Error = (); + type Error = SqlError; - async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_nft_list(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -41,15 +45,15 @@ impl NftListStorageOps for SqliteNftStorage { #[async_trait] impl NftTxHistoryStorageOps for SqliteNftStorage { - type Error = (); + type Error = SqlError; - async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_tx_history(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index b1d42a4910..efc4afcbba 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,7 +1,9 @@ use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTxHistoryStorageOps}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, + NftTxHistoryStorageOps}; use crate::CoinsContext; use async_trait::async_trait; +use derive_more::Display; use mm2_core::mm_ctx::MmArc; pub use mm2_db::indexed_db::InitDbResult; use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder, SharedDb}; @@ -13,6 +15,12 @@ const DB_VERSION: u32 = 1; pub type NftCacheDBLocked<'a> = DbLocked<'a, NftCacheDb>; +#[derive(Debug, Display)] +pub enum WasmNftCacheError {} + +impl NftListStorageError for WasmNftCacheError {} +impl NftTxHistoryStorageError for WasmNftCacheError {} + pub struct NftCacheDb { inner: IndexedDb, } @@ -48,15 +56,15 @@ impl IndexedDbNftStorage { #[async_trait] impl NftListStorageOps for IndexedDbNftStorage { - type Error = (); + type Error = WasmNftCacheError; - async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_nft_list(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -67,15 +75,15 @@ impl NftListStorageOps for IndexedDbNftStorage { #[async_trait] impl NftTxHistoryStorageOps for IndexedDbNftStorage { - type Error = (); + type Error = WasmNftCacheError; - async fn init(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_tx_history(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } - async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, From 67133a24f5898114bd07678c4eff848996f4e9c3 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 11 Apr 2023 18:07:55 +0700 Subject: [PATCH 009/112] change functions in Ops --- mm2src/coins/nft_storage/mod.rs | 8 +++++--- mm2src/coins/nft_storage/sql_storage.rs | 8 +++++--- mm2src/coins/nft_storage/wasm_storage.rs | 8 +++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index ac52aa0fd1..994bfc44d4 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::NotMmError; @@ -20,12 +20,14 @@ pub trait NftListStorageOps { /// Whether tables are initialized for the specified chain. async fn is_initialized_for(&self, chain: Chain) -> MmResult; - async fn get_nft_list(&self, chain: Chain) -> MmResult<(), Self::Error>; + async fn get_nft_list(&self, chain: Chain) -> MmResult; async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; + + async fn remove_nft_from_list(&self, nft: Nft) -> MmResult<(), Self::Error>; } #[async_trait] @@ -38,7 +40,7 @@ pub trait NftTxHistoryStorageOps { /// Whether tables are initialized for the specified chain. async fn is_initialized_for(&self, chain: Chain) -> MmResult; - async fn get_tx_history(&self, chain: Chain) -> MmResult<(), Self::Error>; + async fn get_tx_history(&self, chain: Chain) -> MmResult; async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> where diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index b385644861..bda49cfd32 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; @@ -32,7 +32,7 @@ impl NftListStorageOps for SqliteNftStorage { async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_nft_list(&self, _chain: Chain) -> MmResult { todo!() } async fn add_nfts_to_list(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where @@ -41,6 +41,8 @@ impl NftListStorageOps for SqliteNftStorage { { todo!() } + + async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } } #[async_trait] @@ -51,7 +53,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_tx_history(&self, _chain: Chain) -> MmResult { todo!() } async fn add_txs_to_history(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index efc4afcbba..7656f69e23 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, Nft, NftTransferHistory}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, NftTxHistoryStorageOps}; use crate::CoinsContext; @@ -62,7 +62,7 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_nft_list(&self, _chain: Chain) -> MmResult { todo!() } async fn add_nfts_to_list(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where @@ -71,6 +71,8 @@ impl NftListStorageOps for IndexedDbNftStorage { { todo!() } + + async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } } #[async_trait] @@ -81,7 +83,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn get_tx_history(&self, _chain: Chain) -> MmResult { todo!() } async fn add_txs_to_history(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> where From 2d66f8fae301280b5aa18a050d8e022fd4f389f7 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 12 Apr 2023 14:46:43 +0700 Subject: [PATCH 010/112] wasm: add lock_db fn and From IDB errors --- mm2src/coins/lp_coins.rs | 4 +- mm2src/coins/nft_storage/wasm_storage.rs | 73 ++++++++++++++++++++---- 2 files changed, 65 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index c23254e981..da8dc52402 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -99,7 +99,7 @@ cfg_wasm32! { use tx_history_storage::wasm::{clear_tx_history, load_tx_history, save_tx_history, TxHistoryDb}; pub type TxHistoryDbLocked<'a> = DbLocked<'a, TxHistoryDb>; #[cfg(feature = "enable-nft-integration")] - use nft_storage::wasm_storage::NftCacheDb; + use nft_storage::wasm_storage::NftCacheIDB; } // using custom copy of try_fus as futures crate was renamed to futures01 @@ -2454,7 +2454,7 @@ pub struct CoinsContext { hd_wallet_db: SharedDb, #[cfg(feature = "enable-nft-integration")] #[cfg(target_arch = "wasm32")] - pub nft_cache_db: SharedDb, + pub nft_cache_db: SharedDb, } #[derive(Debug)] diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index 7656f69e23..94827689e5 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -6,43 +6,92 @@ use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; pub use mm2_db::indexed_db::InitDbResult; -use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder, SharedDb}; +use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, DbTransactionError, IndexedDb, IndexedDbBuilder, + InitDbError, SharedDb}; +use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; -pub type NftCacheDBLocked<'a> = DbLocked<'a, NftCacheDb>; - -#[derive(Debug, Display)] -pub enum WasmNftCacheError {} +pub type NftCacheIDBResult = MmResult; +pub type NftCacheIDBLocked<'a> = DbLocked<'a, NftCacheIDB>; impl NftListStorageError for WasmNftCacheError {} impl NftTxHistoryStorageError for WasmNftCacheError {} -pub struct NftCacheDb { +#[derive(Debug, Display)] +pub enum WasmNftCacheError { + ErrorSerializing(String), + ErrorDeserializing(String), + ErrorSaving(String), + ErrorLoading(String), + ErrorClearing(String), + NotSupported(String), + InternalError(String), +} + +impl From for WasmNftCacheError { + fn from(e: InitDbError) -> Self { + match &e { + InitDbError::NotSupported(_) => WasmNftCacheError::NotSupported(e.to_string()), + InitDbError::EmptyTableList + | InitDbError::DbIsOpenAlready { .. } + | InitDbError::InvalidVersion(_) + | InitDbError::OpeningError(_) + | InitDbError::TypeMismatch { .. } + | InitDbError::UnexpectedState(_) + | InitDbError::UpgradingError { .. } => WasmNftCacheError::InternalError(e.to_string()), + } + } +} + +impl From for WasmNftCacheError { + fn from(e: DbTransactionError) -> Self { + match e { + DbTransactionError::ErrorSerializingItem(_) => WasmNftCacheError::ErrorSerializing(e.to_string()), + DbTransactionError::ErrorDeserializingItem(_) => WasmNftCacheError::ErrorDeserializing(e.to_string()), + DbTransactionError::ErrorUploadingItem(_) => WasmNftCacheError::ErrorSaving(e.to_string()), + DbTransactionError::ErrorGettingItems(_) | DbTransactionError::ErrorCountingItems(_) => { + WasmNftCacheError::ErrorLoading(e.to_string()) + }, + DbTransactionError::ErrorDeletingItems(_) => WasmNftCacheError::ErrorClearing(e.to_string()), + DbTransactionError::NoSuchTable { .. } + | DbTransactionError::ErrorCreatingTransaction(_) + | DbTransactionError::ErrorOpeningTable { .. } + | DbTransactionError::ErrorSerializingIndex { .. } + | DbTransactionError::UnexpectedState(_) + | DbTransactionError::TransactionAborted + | DbTransactionError::MultipleItemsByUniqueIndex { .. } + | DbTransactionError::NoSuchIndex { .. } + | DbTransactionError::InvalidIndex { .. } => WasmNftCacheError::InternalError(e.to_string()), + } + } +} + +pub struct NftCacheIDB { inner: IndexedDb, } #[async_trait] -impl DbInstance for NftCacheDb { +impl DbInstance for NftCacheIDB { fn db_name() -> &'static str { DB_NAME } async fn init(db_id: DbIdentifier) -> InitDbResult { // todo add tables for each chain? let inner = IndexedDbBuilder::new(db_id).with_version(DB_VERSION).build().await?; - Ok(NftCacheDb { inner }) + Ok(NftCacheIDB { inner }) } } -impl NftCacheDb { +impl NftCacheIDB { fn get_inner(&self) -> &IndexedDb { &self.inner } } #[derive(Clone)] pub struct IndexedDbNftStorage { - db: SharedDb, + db: SharedDb, } impl IndexedDbNftStorage { @@ -52,6 +101,10 @@ impl IndexedDbNftStorage { db: coins_ctx.nft_cache_db.clone(), }) } + + async fn lock_db(&self) -> NftCacheIDBResult> { + self.db.get_or_initialize().await.mm_err(WasmNftCacheError::from) + } } #[async_trait] From 9225e99f9369d4c4b506a244a35ee94adf793622 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 12 Apr 2023 14:53:40 +0700 Subject: [PATCH 011/112] wasm: change name --- mm2src/coins/nft_storage/wasm_storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index 94827689e5..c6b04bbbb9 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -15,7 +15,7 @@ use mm2_err_handle::prelude::MmResult; const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; -pub type NftCacheIDBResult = MmResult; +pub type WasmNftCacheResult = MmResult; pub type NftCacheIDBLocked<'a> = DbLocked<'a, NftCacheIDB>; impl NftListStorageError for WasmNftCacheError {} @@ -102,7 +102,7 @@ impl IndexedDbNftStorage { }) } - async fn lock_db(&self) -> NftCacheIDBResult> { + async fn lock_db(&self) -> WasmNftCacheResult> { self.db.get_or_initialize().await.mm_err(WasmNftCacheError::from) } } From 9f977f314b40d0a7ab8789e59912210ee9b482c2 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 12 Apr 2023 21:59:34 +0700 Subject: [PATCH 012/112] sql: table names --- mm2src/coins/nft_storage/sql_storage.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index bda49cfd32..a75999927a 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; @@ -7,6 +7,10 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use std::sync::{Arc, Mutex}; +fn nft_list_table(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } + +fn nft_tx_history_table(chain: &Chain) -> String { chain.to_ticker() + "_nft_tx_history" } + impl NftListStorageError for SqlError {} impl NftTxHistoryStorageError for SqlError {} From de5cff6d12f1aa85a62664ab3dd8c014c940c9b6 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 13 Apr 2023 16:41:48 +0700 Subject: [PATCH 013/112] sql: save create table commands --- mm2src/coins/nft_storage/sql_storage.rs | 58 ++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index a75999927a..ffb07c7123 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -7,9 +7,63 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use std::sync::{Arc, Mutex}; -fn nft_list_table(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } +fn nft_list_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } + +fn nft_tx_history_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_tx_history" } + +fn create_nft_list_table_sql(chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + chain TEXT NOT NULL, + token_address TEXT NOT NULL, + token_id VARCHAR(256) NOT NULL, + amount VARCHAR(256) NOT NULL, + owner_of TEXT NOT NULL, + token_hash TEXT NOT NULL, + block_number_minted INTEGER NOT NULL, + block_number INTEGER NOT NULL, + contract_type TEXT, + name TEXT, + symbol TEXT, + token_uri TEXT, + metadata BLOB, + last_token_uri_sync TEXT, + last_metadata_sync TEXT, + minter_address TEXT, + PRIMARY KEY (token_address, token_id) + );", + table_name + ); + Ok(sql) +} -fn nft_tx_history_table(chain: &Chain) -> String { chain.to_ticker() + "_nft_tx_history" } +fn create_tx_history_table_sql(chain: &Chain) -> MmResult { + let table_name = nft_tx_history_table_name(chain); + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + chain TEXT NOT NULL, + block_number INTEGER NOT NULL, + block_timestamp TEXT NOT NULL, + block_hash TEXT NOT NULL, + transaction_hash VARCHAR(256) PRIMARY KEY, + transaction_index INTEGER NOT NULL, + log_index INTEGER NOT NULL, + value VARCHAR(256) NOT NULL, + contract_type TEXT NOT NULL, + transaction_type TEXT NOT NULL, + token_address TEXT NOT NULL, + token_id VARCHAR(256) NOT NULL, + from_address TEXT NOT NULL, + to_address TEXT NOT NULL, + amount VARCHAR(256) NOT NULL, + verified INTEGER NOT NULL, + operator TEXT + );", + table_name + ); + Ok(sql) +} impl NftListStorageError for SqlError {} impl NftTxHistoryStorageError for SqlError {} From 65c702b7d134f8b63673b16a4001423e766eb68e Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 13 Apr 2023 19:44:40 +0700 Subject: [PATCH 014/112] sql: init fnc --- mm2src/coins/nft/nft_structs.rs | 21 +++++++++++ mm2src/coins/nft_storage/mod.rs | 16 ++++----- mm2src/coins/nft_storage/sql_storage.rs | 46 ++++++++++++++++++------ mm2src/coins/nft_storage/wasm_storage.rs | 16 ++++----- 4 files changed, 73 insertions(+), 26 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 70e39c780c..cd6b443940 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -54,6 +54,27 @@ impl ConvertChain for Chain { } } +#[derive(Debug, Display)] +pub enum ParseChainTypeError { + UnsupportedCainType, +} + +impl FromStr for Chain { + type Err = ParseChainTypeError; + + #[inline] + fn from_str(s: &str) -> Result { + match s { + "avalanche" => Ok(Chain::Avalanche), + "bsc" => Ok(Chain::Bsc), + "eth" => Ok(Chain::Eth), + "fantom" => Ok(Chain::Fantom), + "polygon" => Ok(Chain::Polygon), + _ => Err(ParseChainTypeError::UnsupportedCainType), + } + } +} + #[derive(Debug, Display)] pub(crate) enum ParseContractTypeError { UnsupportedContractType, diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 994bfc44d4..ca64d2fb5e 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -15,14 +15,14 @@ pub trait NftListStorageOps { type Error: NftListStorageError; /// Initializes tables in storage for the specified chain type. - async fn init(&self, chain: Chain) -> MmResult<(), Self::Error>; + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. - async fn is_initialized_for(&self, chain: Chain) -> MmResult; + async fn is_initialized_for(&self, chain: &Chain) -> MmResult; - async fn get_nft_list(&self, chain: Chain) -> MmResult; + async fn get_nft_list(&self, chain: &Chain) -> MmResult; - async fn add_nfts_to_list(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; @@ -35,14 +35,14 @@ pub trait NftTxHistoryStorageOps { type Error: NftTxHistoryStorageError; /// Initializes tables in storage for the specified chain type. - async fn init(&self, chain: Chain) -> MmResult<(), Self::Error>; + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. - async fn is_initialized_for(&self, chain: Chain) -> MmResult; + async fn is_initialized_for(&self, chain: &Chain) -> MmResult; - async fn get_tx_history(&self, chain: Chain) -> MmResult; + async fn get_tx_history(&self, chain: &Chain) -> MmResult; - async fn add_txs_to_history(&self, chain: Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index ffb07c7123..da134f1475 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -2,17 +2,23 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHist use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError}; +use common::async_blocking; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; +use db_common::sqlite::validate_table_name; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use std::sync::{Arc, Mutex}; +#[allow(dead_code)] fn nft_list_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } +#[allow(dead_code)] fn nft_tx_history_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_tx_history" } +#[allow(dead_code)] fn create_nft_list_table_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( chain TEXT NOT NULL, @@ -38,15 +44,17 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { Ok(sql) } +#[allow(dead_code)] fn create_tx_history_table_sql(chain: &Chain) -> MmResult { let table_name = nft_tx_history_table_name(chain); + validate_table_name(&table_name)?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( + transaction_hash VARCHAR(256) PRIMARY KEY, chain TEXT NOT NULL, block_number INTEGER NOT NULL, block_timestamp TEXT NOT NULL, block_hash TEXT NOT NULL, - transaction_hash VARCHAR(256) PRIMARY KEY, transaction_index INTEGER NOT NULL, log_index INTEGER NOT NULL, value VARCHAR(256) NOT NULL, @@ -86,13 +94,22 @@ impl SqliteNftStorage { impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; - async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + let sql_nft_list = create_nft_list_table_sql(chain)?; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + conn.execute(&sql_nft_list, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } - async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } + async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, _chain: Chain) -> MmResult { todo!() } + async fn get_nft_list(&self, _chain: &Chain) -> MmResult { todo!() } - async fn add_nfts_to_list(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -107,13 +124,22 @@ impl NftListStorageOps for SqliteNftStorage { impl NftTxHistoryStorageOps for SqliteNftStorage { type Error = SqlError; - async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + let sql_tx_history = create_tx_history_table_sql(chain)?; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + conn.execute(&sql_tx_history, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } - async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } + async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, _chain: Chain) -> MmResult { todo!() } + async fn get_tx_history(&self, _chain: &Chain) -> MmResult { todo!() } - async fn add_txs_to_history(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index c6b04bbbb9..b7b331cd9d 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -111,13 +111,13 @@ impl IndexedDbNftStorage { impl NftListStorageOps for IndexedDbNftStorage { type Error = WasmNftCacheError; - async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } + async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, _chain: Chain) -> MmResult { todo!() } + async fn get_nft_list(&self, _chain: &Chain) -> MmResult { todo!() } - async fn add_nfts_to_list(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -132,13 +132,13 @@ impl NftListStorageOps for IndexedDbNftStorage { impl NftTxHistoryStorageOps for IndexedDbNftStorage { type Error = WasmNftCacheError; - async fn init(&self, _chain: Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, _chain: Chain) -> MmResult { todo!() } + async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, _chain: Chain) -> MmResult { todo!() } + async fn get_tx_history(&self, _chain: &Chain) -> MmResult { todo!() } - async fn add_txs_to_history(&self, _chain: Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, From 52b3c325c75929a0dc24c0a9b0754a80a293d6ce Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 13 Apr 2023 20:00:19 +0700 Subject: [PATCH 015/112] sql: is_initialized_for fnc --- mm2src/coins/nft_storage/sql_storage.rs | 27 ++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index da134f1475..11cc8887c3 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -4,7 +4,7 @@ use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStor use async_trait::async_trait; use common::async_blocking; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; -use db_common::sqlite::validate_table_name; +use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use std::sync::{Arc, Mutex}; @@ -105,7 +105,17 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } + async fn is_initialized_for(&self, chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let nft_list_initialized = query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [table_name], string_from_row)?; + Ok(nft_list_initialized.is_some()) + }) + .await + } async fn get_nft_list(&self, _chain: &Chain) -> MmResult { todo!() } @@ -135,7 +145,18 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } + async fn is_initialized_for(&self, chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let tx_history_initialized = + query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [table_name], string_from_row)?; + Ok(tx_history_initialized.is_some()) + }) + .await + } async fn get_tx_history(&self, _chain: &Chain) -> MmResult { todo!() } From 6d34df54f548147dcbab22d6f1b64c2e603ea910 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 20 Apr 2023 16:07:32 +0700 Subject: [PATCH 016/112] remove wip, leave draft --- mm2src/coins/nft/nft_errors.rs | 28 ++++++++++++++++++++++++- mm2src/coins/nft/nft_structs.rs | 11 ++++++++++ mm2src/coins/nft_storage/mod.rs | 3 +++ mm2src/coins/nft_storage/sql_storage.rs | 4 ++-- 4 files changed, 43 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index bd510108ab..fe82959061 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,5 +1,10 @@ use crate::eth::GetEthAddressError; +#[cfg(target_arch = "wasm32")] +use crate::nft_storage::wasm_storage::WasmNftCacheError; +use crate::nft_storage::CreateNftStorageError; use common::HttpStatusCode; +#[cfg(not(target_arch = "wasm32"))] +use db_common::sqlite::rusqlite::Error as SqlError; use derive_more::Display; use enum_from::EnumFromStringify; use http::StatusCode; @@ -33,6 +38,8 @@ pub enum GetNftInfoError { token_address: String, token_id: String, }, + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl From for GetNftInfoError { @@ -63,6 +70,24 @@ impl From for GetNftInfoError { fn from(e: GetEthAddressError) -> Self { GetNftInfoError::GetEthAddressError(e) } } +impl From for GetNftInfoError { + fn from(e: CreateNftStorageError) -> Self { + match e { + CreateNftStorageError::Internal(err) => GetNftInfoError::Internal(err), + } + } +} + +#[cfg(not(target_arch = "wasm32"))] +impl From for GetNftInfoError { + fn from(e: SqlError) -> Self { GetNftInfoError::DbError(e.to_string()) } +} + +#[cfg(target_arch = "wasm32")] +impl From for GetNftInfoError { + fn from(e: WasmNftCacheError) -> Self { GetNftInfoError::DbError(e.to_string()) } +} + impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { @@ -72,7 +97,8 @@ impl HttpStatusCode for GetNftInfoError { GetNftInfoError::Transport(_) | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) - | GetNftInfoError::TokenNotFoundInWallet { .. } => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::TokenNotFoundInWallet { .. } + | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index cd6b443940..74ec49b0bd 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -276,3 +276,14 @@ pub(crate) struct NftTransferHistoryWrapper { pub struct NftsTransferHistoryList { pub(crate) transfer_history: Vec, } + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct GetNftTxHistoryFilters { + from_block: Option, + to_block: Option, + from_address: Option, + to_address: Option, + from_date: Option, + to_date: Option, +} diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index ca64d2fb5e..48ae6c4d46 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,8 +1,10 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use async_trait::async_trait; +use derive_more::Display; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::NotMmError; use mm2_err_handle::prelude::MmResult; +use serde::{Deserialize, Serialize}; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; @@ -48,6 +50,7 @@ pub trait NftTxHistoryStorageOps { I::IntoIter: Send; } +#[derive(Debug, Deserialize, Display, Serialize)] pub enum CreateNftStorageError { Internal(String), } diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 11cc8887c3..914faef9dc 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -22,7 +22,7 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( chain TEXT NOT NULL, - token_address TEXT NOT NULL, + token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, amount VARCHAR(256) NOT NULL, owner_of TEXT NOT NULL, @@ -60,7 +60,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { value VARCHAR(256) NOT NULL, contract_type TEXT NOT NULL, transaction_type TEXT NOT NULL, - token_address TEXT NOT NULL, + token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, from_address TEXT NOT NULL, to_address TEXT NOT NULL, From 9bf64534ad76902f2fe0b98ecef88f4074b62a95 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 21 Apr 2023 16:03:01 +0700 Subject: [PATCH 017/112] add possible_spam --- mm2src/coins/nft.rs | 3 +++ mm2src/coins/nft/nft_structs.rs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 2b768738bd..c17f6cd25a 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -66,6 +66,7 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult MmResult, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, + pub(crate) possible_spam: Option, } /// This structure is for deserializing NFT json to struct. @@ -139,6 +140,7 @@ pub(crate) struct NftWrapper { pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, + pub(crate) possible_spam: Option, } #[derive(Debug)] @@ -249,6 +251,7 @@ pub struct NftTransferHistory { pub(crate) amount: BigDecimal, pub(crate) verified: u64, pub(crate) operator: Option, + pub(crate) possible_spam: Option, } #[derive(Debug, Deserialize)] @@ -270,6 +273,7 @@ pub(crate) struct NftTransferHistoryWrapper { pub(crate) amount: SerdeStringWrap, pub(crate) verified: u64, pub(crate) operator: Option, + pub(crate) possible_spam: Option, } #[derive(Debug, Serialize)] From 0a19dba5d7bb8b1a6b065a8d71d9643d2989ef60 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 23 Apr 2023 21:04:05 +0700 Subject: [PATCH 018/112] unite traits --- mm2src/coins/nft.rs | 70 +++------------- mm2src/coins/nft/nft_errors.rs | 23 ++++-- mm2src/coins/nft_storage/mod.rs | 101 ++++++++++++++++++------ mm2src/coins/nft_storage/sql_storage.rs | 59 +++++++------- 4 files changed, 137 insertions(+), 116 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index c17f6cd25a..bb55e7c34f 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -6,81 +6,35 @@ pub(crate) mod nft_structs; use crate::WithdrawError; use nft_errors::GetNftInfoError; -use nft_structs::{Chain, ConvertChain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, - NftTransferHistoryWrapper, NftTransfersReq, NftWrapper, NftsTransferHistoryList, - TransactionNftDetails, WithdrawNftReq}; +use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, NftTransferHistoryWrapper, + NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; +use crate::nft_storage::{NftStorageOps, NftStorageBuilder}; use common::{APPLICATION_JSON, X_API_KEY}; use http::header::ACCEPT; use mm2_number::BigDecimal; use serde_json::Value as Json; /// url for moralis requests -const URL_MORALIS: &str = "https://deep-index.moralis.io/api/v2/"; +pub const URL_MORALIS: &str = "https://deep-index.moralis.io/api/v2/"; /// query parameter for moralis request: The format of the token ID -const FORMAT_DECIMAL_MORALIS: &str = "format=decimal"; +pub const FORMAT_DECIMAL_MORALIS: &str = "format=decimal"; /// query parameter for moralis request: The transfer direction -const DIRECTION_BOTH_MORALIS: &str = "direction=both"; +pub const DIRECTION_BOTH_MORALIS: &str = "direction=both"; pub type WithdrawNftResult = Result>; /// `get_nft_list` function returns list of NFTs on requested chains owned by user. pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult { - let api_key = ctx.conf["api_key"] - .as_str() - .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - let mut res_list = Vec::new(); - + let storage = NftStorageBuilder::new(&ctx).build()?; for chain in req.chains { - let (coin_str, chain_str) = chain.to_ticker_chain(); - let my_address = get_eth_address(&ctx, &coin_str).await?; - let uri_without_cursor = format!( - "{}{}/nft?chain={}&{}", - URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS - ); - - // The cursor returned in the previous response (used for getting the next page). - let mut cursor = String::new(); - loop { - let uri = format!("{}{}", uri_without_cursor, cursor); - let response = send_moralis_request(uri.as_str(), api_key).await?; - if let Some(nfts_list) = response["result"].as_array() { - for nft_json in nfts_list { - let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; - let nft = Nft { - chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, - block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.map(|v| v.0), - name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, - }; - // collect NFTs from the page - res_list.push(nft); - } - // if cursor is not null, there are other NFTs on next page, - // and we need to send new request with cursor to get info from the next page. - if let Some(cursor_res) = response["cursor"].as_str() { - cursor = format!("{}{}", "&cursor=", cursor_res); - continue; - } else { - break; - } - } + if storage.is_initialized_for_list(&chain).await? { + storage.init_list(&chain).await?; } + let nfts = storage.get_nft_list(&ctx, &chain).await?; + res_list.extend(nfts); } drop_mutability!(res_list); let nft_list = NftList { nfts: res_list }; @@ -217,7 +171,7 @@ pub async fn withdraw_nft(ctx: MmArc, req_type: WithdrawNftReq) -> WithdrawNftRe } #[cfg(not(target_arch = "wasm32"))] -async fn send_moralis_request(uri: &str, api_key: &str) -> MmResult { +pub async fn send_moralis_request(uri: &str, api_key: &str) -> MmResult { use http::header::HeaderValue; use mm2_net::transport::slurp_req_body; diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index fe82959061..2add18ab42 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,7 +1,7 @@ use crate::eth::GetEthAddressError; #[cfg(target_arch = "wasm32")] use crate::nft_storage::wasm_storage::WasmNftCacheError; -use crate::nft_storage::CreateNftStorageError; +use crate::nft_storage::{CreateNftStorageError, NftStorageError}; use common::HttpStatusCode; #[cfg(not(target_arch = "wasm32"))] use db_common::sqlite::rusqlite::Error as SqlError; @@ -78,15 +78,22 @@ impl From for GetNftInfoError { } } -#[cfg(not(target_arch = "wasm32"))] -impl From for GetNftInfoError { - fn from(e: SqlError) -> Self { GetNftInfoError::DbError(e.to_string()) } +impl From for GetNftInfoError { + fn from(err: T) -> Self { + let msg = format!("{:?}", err); + GetNftInfoError::DbError(msg) + } } -#[cfg(target_arch = "wasm32")] -impl From for GetNftInfoError { - fn from(e: WasmNftCacheError) -> Self { GetNftInfoError::DbError(e.to_string()) } -} +// #[cfg(not(target_arch = "wasm32"))] +// impl From for GetNftInfoError { +// fn from(e: SqlError) -> Self { GetNftInfoError::DbError(e.to_string()) } +// } +// +// #[cfg(target_arch = "wasm32")] +// impl From for GetNftInfoError { +// fn from(e: WasmNftCacheError) -> Self { GetNftInfoError::DbError(e.to_string()) } +// } impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 48ae6c4d46..5da08c183d 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,28 +1,37 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; +use crate::eth::get_eth_address; +use crate::nft::nft_errors::GetNftInfoError; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftTransferHistory, NftWrapper}; +use crate::nft::{send_moralis_request, FORMAT_DECIMAL_MORALIS, URL_MORALIS}; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::NotMmError; -use mm2_err_handle::prelude::MmResult; +use mm2_err_handle::mm_error::{MmError, MmResult}; use serde::{Deserialize, Serialize}; +use std::format; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; -pub trait NftListStorageError: std::fmt::Debug + NotMmError + Send {} -pub trait NftTxHistoryStorageError: std::fmt::Debug + NotMmError + Send {} +pub trait NftStorageError: std::fmt::Debug + NotMmError + Send {} #[async_trait] -pub trait NftListStorageOps { - type Error: NftListStorageError; +pub trait NftStorageOps { + type Error: NftStorageError; /// Initializes tables in storage for the specified chain type. - async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; + async fn init_list(&self, chain: &Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. - async fn is_initialized_for(&self, chain: &Chain) -> MmResult; + async fn is_initialized_for_list(&self, chain: &Chain) -> MmResult; - async fn get_nft_list(&self, chain: &Chain) -> MmResult; + /// Initializes tables in storage for the specified chain type. + async fn init_history(&self, chain: &Chain) -> MmResult<(), Self::Error>; + + /// Whether tables are initialized for the specified chain. + async fn is_initialized_for_history(&self, chain: &Chain) -> MmResult; + + async fn get_nft_list(&self, ctx: &MmArc, chain: &Chain) -> MmResult, Self::Error>; async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where @@ -30,19 +39,8 @@ pub trait NftListStorageOps { I::IntoIter: Send; async fn remove_nft_from_list(&self, nft: Nft) -> MmResult<(), Self::Error>; -} - -#[async_trait] -pub trait NftTxHistoryStorageOps { - type Error: NftTxHistoryStorageError; - /// Initializes tables in storage for the specified chain type. - async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; - - /// Whether tables are initialized for the specified chain. - async fn is_initialized_for(&self, chain: &Chain) -> MmResult; - - async fn get_tx_history(&self, chain: &Chain) -> MmResult; + async fn get_tx_history(&self, ctx: &MmArc, chain: &Chain) -> MmResult, Self::Error>; async fn add_txs_to_history(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where @@ -66,10 +64,69 @@ impl<'a> NftStorageBuilder<'a> { pub fn new(ctx: &MmArc) -> NftStorageBuilder<'_> { NftStorageBuilder { ctx } } #[inline] - pub fn build(self) -> MmResult { + pub fn build(self) -> MmResult { #[cfg(target_arch = "wasm32")] return wasm_storage::IndexedDbNftStorage::new(self.ctx); #[cfg(not(target_arch = "wasm32"))] sql_storage::SqliteNftStorage::new(self.ctx) } } + +async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, GetNftInfoError> { + let api_key = ctx.conf["api_key"] + .as_str() + .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; + + let mut res_list = Vec::new(); + + let (coin_str, chain_str) = chain.to_ticker_chain(); + let my_address = get_eth_address(&ctx, &coin_str).await?; + let uri_without_cursor = format!( + "{}{}/nft?chain={}&{}", + URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS + ); + + // The cursor returned in the previous response (used for getting the next page). + let mut cursor = String::new(); + loop { + let uri = format!("{}{}", uri_without_cursor, cursor); + let response = send_moralis_request(uri.as_str(), api_key).await?; + if let Some(nfts_list) = response["result"].as_array() { + for nft_json in nfts_list { + let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; + let nft = Nft { + chain: chain.clone(), + token_address: nft_wrapper.token_address, + token_id: nft_wrapper.token_id.0, + amount: nft_wrapper.amount.0, + owner_of: nft_wrapper.owner_of, + token_hash: nft_wrapper.token_hash, + block_number_minted: *nft_wrapper.block_number_minted, + block_number: *nft_wrapper.block_number, + contract_type: nft_wrapper.contract_type.map(|v| v.0), + name: nft_wrapper.name, + symbol: nft_wrapper.symbol, + token_uri: nft_wrapper.token_uri, + metadata: nft_wrapper.metadata, + last_token_uri_sync: nft_wrapper.last_token_uri_sync, + last_metadata_sync: nft_wrapper.last_metadata_sync, + minter_address: nft_wrapper.minter_address, + possible_spam: nft_wrapper.possible_spam, + }; + // collect NFTs from the page + res_list.push(nft); + } + // if cursor is not null, there are other NFTs on next page, + // and we need to send new request with cursor to get info from the next page. + if let Some(cursor_res) = response["cursor"].as_str() { + cursor = format!("{}{}", "&cursor=", cursor_res); + continue; + } else { + break; + } + } + } + + drop_mutability!(res_list); + Ok(res_list) +} diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 914faef9dc..e09c883f8c 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,6 +1,5 @@ -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, - NftTxHistoryStorageOps}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftTransferHistory}; +use crate::nft_storage::{CreateNftStorageError, NftStorageError, NftStorageOps}; use async_trait::async_trait; use common::async_blocking; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; @@ -73,8 +72,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { Ok(sql) } -impl NftListStorageError for SqlError {} -impl NftTxHistoryStorageError for SqlError {} +impl NftStorageError for SqlError {} #[derive(Clone)] pub struct SqliteNftStorage(Arc>); @@ -91,10 +89,10 @@ impl SqliteNftStorage { } #[async_trait] -impl NftListStorageOps for SqliteNftStorage { +impl NftStorageOps for SqliteNftStorage { type Error = SqlError; - async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { + async fn init_list(&self, chain: &Chain) -> MmResult<(), Self::Error> { let selfi = self.clone(); let sql_nft_list = create_nft_list_table_sql(chain)?; async_blocking(move || { @@ -105,7 +103,7 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn is_initialized_for(&self, chain: &Chain) -> MmResult { + async fn is_initialized_for_list(&self, chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; let selfi = self.clone(); @@ -117,24 +115,7 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn get_nft_list(&self, _chain: &Chain) -> MmResult { todo!() } - - async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> - where - I: IntoIterator + Send + 'static, - I::IntoIter: Send, - { - todo!() - } - - async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } -} - -#[async_trait] -impl NftTxHistoryStorageOps for SqliteNftStorage { - type Error = SqlError; - - async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { + async fn init_history(&self, chain: &Chain) -> MmResult<(), Self::Error> { let selfi = self.clone(); let sql_tx_history = create_tx_history_table_sql(chain)?; async_blocking(move || { @@ -145,7 +126,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn is_initialized_for(&self, chain: &Chain) -> MmResult { + async fn is_initialized_for_history(&self, chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; let selfi = self.clone(); @@ -158,7 +139,29 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn get_tx_history(&self, _chain: &Chain) -> MmResult { todo!() } + async fn get_nft_list(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + { + let selfi = self.clone(); + let _chain = chain.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let _sql_transaction = conn.transaction()?; + + Ok(()) + }) + .await + } + + async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } + + async fn get_tx_history(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { + todo!() + } async fn add_txs_to_history(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where From 9e2b935b5b0811f370343411dc0752214307537f Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 23 Apr 2023 21:32:33 +0700 Subject: [PATCH 019/112] fmt --- mm2src/coins/nft.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index bb55e7c34f..96ea24189d 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -10,7 +10,7 @@ use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHi NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft_storage::{NftStorageOps, NftStorageBuilder}; +use crate::nft_storage::{NftStorageBuilder, NftStorageOps}; use common::{APPLICATION_JSON, X_API_KEY}; use http::header::ACCEPT; use mm2_number::BigDecimal; From 8349de1e39921d1ce340367dc05b052d22f88f1c Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 23 Apr 2023 22:06:35 +0700 Subject: [PATCH 020/112] wip --- mm2src/coins/nft/nft_errors.rs | 14 -------------- mm2src/coins/nft_storage/mod.rs | 9 +++++---- mm2src/coins/nft_storage/sql_storage.rs | 4 ++-- 3 files changed, 7 insertions(+), 20 deletions(-) diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 2add18ab42..27d66fdfc1 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,10 +1,6 @@ use crate::eth::GetEthAddressError; -#[cfg(target_arch = "wasm32")] -use crate::nft_storage::wasm_storage::WasmNftCacheError; use crate::nft_storage::{CreateNftStorageError, NftStorageError}; use common::HttpStatusCode; -#[cfg(not(target_arch = "wasm32"))] -use db_common::sqlite::rusqlite::Error as SqlError; use derive_more::Display; use enum_from::EnumFromStringify; use http::StatusCode; @@ -85,16 +81,6 @@ impl From for GetNftInfoError { } } -// #[cfg(not(target_arch = "wasm32"))] -// impl From for GetNftInfoError { -// fn from(e: SqlError) -> Self { GetNftInfoError::DbError(e.to_string()) } -// } -// -// #[cfg(target_arch = "wasm32")] -// impl From for GetNftInfoError { -// fn from(e: WasmNftCacheError) -> Self { GetNftInfoError::DbError(e.to_string()) } -// } - impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 5da08c183d..695032514c 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -5,15 +5,15 @@ use crate::nft::{send_moralis_request, FORMAT_DECIMAL_MORALIS, URL_MORALIS}; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; -use mm2_err_handle::mm_error::NotMmError; use mm2_err_handle::mm_error::{MmError, MmResult}; +use mm2_err_handle::mm_error::{NotEqual, NotMmError}; use serde::{Deserialize, Serialize}; use std::format; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; -pub trait NftStorageError: std::fmt::Debug + NotMmError + Send {} +pub trait NftStorageError: std::fmt::Debug + NotMmError + NotEqual + Send {} #[async_trait] pub trait NftStorageOps { @@ -72,6 +72,7 @@ impl<'a> NftStorageBuilder<'a> { } } +#[allow(dead_code)] async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, GetNftInfoError> { let api_key = ctx.conf["api_key"] .as_str() @@ -80,7 +81,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, let mut res_list = Vec::new(); let (coin_str, chain_str) = chain.to_ticker_chain(); - let my_address = get_eth_address(&ctx, &coin_str).await?; + let my_address = get_eth_address(ctx, &coin_str).await?; let uri_without_cursor = format!( "{}{}/nft?chain={}&{}", URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS @@ -95,7 +96,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, for nft_json in nfts_list { let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; let nft = Nft { - chain: chain.clone(), + chain: *chain, token_address: nft_wrapper.token_address, token_id: nft_wrapper.token_id.0, amount: nft_wrapper.amount.0, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index e09c883f8c..9426138fc5 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -141,13 +141,13 @@ impl NftStorageOps for SqliteNftStorage { async fn get_nft_list(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { todo!() } - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { let selfi = self.clone(); - let _chain = chain.clone(); + let _chain = *chain; async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let _sql_transaction = conn.transaction()?; From d2e0653933475d9cd77fb14ce83c8ae476342109 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 24 Apr 2023 09:16:40 +0700 Subject: [PATCH 021/112] save wip state, moralis was removed from get_nft_list --- mm2src/coins/nft.rs | 6 +-- mm2src/coins/nft_storage/mod.rs | 25 ++++++----- mm2src/coins/nft_storage/sql_storage.rs | 60 +++++++++++++------------ 3 files changed, 50 insertions(+), 41 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 96ea24189d..66848297a7 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -10,7 +10,7 @@ use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHi NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft_storage::{NftStorageBuilder, NftStorageOps}; +use crate::nft_storage::{NftListStorageOps, NftStorageBuilder}; use common::{APPLICATION_JSON, X_API_KEY}; use http::header::ACCEPT; use mm2_number::BigDecimal; @@ -30,8 +30,8 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult<(), Self::Error>; + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; /// Whether tables are initialized for the specified chain. - async fn is_initialized_for_list(&self, chain: &Chain) -> MmResult; - - /// Initializes tables in storage for the specified chain type. - async fn init_history(&self, chain: &Chain) -> MmResult<(), Self::Error>; - - /// Whether tables are initialized for the specified chain. - async fn is_initialized_for_history(&self, chain: &Chain) -> MmResult; + async fn is_initialized(&self, chain: &Chain) -> MmResult; async fn get_nft_list(&self, ctx: &MmArc, chain: &Chain) -> MmResult, Self::Error>; @@ -39,6 +33,17 @@ pub trait NftStorageOps { I::IntoIter: Send; async fn remove_nft_from_list(&self, nft: Nft) -> MmResult<(), Self::Error>; +} + +#[async_trait] +pub trait NftTxHistoryStorageOps { + type Error: NftStorageError; + + /// Initializes tables in storage for the specified chain type. + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error>; + + /// Whether tables are initialized for the specified chain. + async fn is_initialized(&self, chain: &Chain) -> MmResult; async fn get_tx_history(&self, ctx: &MmArc, chain: &Chain) -> MmResult, Self::Error>; @@ -64,7 +69,7 @@ impl<'a> NftStorageBuilder<'a> { pub fn new(ctx: &MmArc) -> NftStorageBuilder<'_> { NftStorageBuilder { ctx } } #[inline] - pub fn build(self) -> MmResult { + pub fn build(self) -> MmResult { #[cfg(target_arch = "wasm32")] return wasm_storage::IndexedDbNftStorage::new(self.ctx); #[cfg(not(target_arch = "wasm32"))] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 9426138fc5..f5c881e985 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,5 +1,5 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftTransferHistory}; -use crate::nft_storage::{CreateNftStorageError, NftStorageError, NftStorageOps}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; use common::async_blocking; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; @@ -89,10 +89,10 @@ impl SqliteNftStorage { } #[async_trait] -impl NftStorageOps for SqliteNftStorage { +impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; - async fn init_list(&self, chain: &Chain) -> MmResult<(), Self::Error> { + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { let selfi = self.clone(); let sql_nft_list = create_nft_list_table_sql(chain)?; async_blocking(move || { @@ -103,7 +103,7 @@ impl NftStorageOps for SqliteNftStorage { .await } - async fn is_initialized_for_list(&self, chain: &Chain) -> MmResult { + async fn is_initialized(&self, chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; let selfi = self.clone(); @@ -115,30 +115,6 @@ impl NftStorageOps for SqliteNftStorage { .await } - async fn init_history(&self, chain: &Chain) -> MmResult<(), Self::Error> { - let selfi = self.clone(); - let sql_tx_history = create_tx_history_table_sql(chain)?; - async_blocking(move || { - let conn = selfi.0.lock().unwrap(); - conn.execute(&sql_tx_history, NO_PARAMS).map(|_| ())?; - Ok(()) - }) - .await - } - - async fn is_initialized_for_history(&self, chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; - let selfi = self.clone(); - async_blocking(move || { - let conn = selfi.0.lock().unwrap(); - let tx_history_initialized = - query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [table_name], string_from_row)?; - Ok(tx_history_initialized.is_some()) - }) - .await - } - async fn get_nft_list(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { todo!() } async fn add_nfts_to_list(&self, chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> @@ -158,6 +134,34 @@ impl NftStorageOps for SqliteNftStorage { } async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } +} + +#[async_trait] +impl NftTxHistoryStorageOps for SqliteNftStorage { + type Error = SqlError; + + async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + let sql_nft_list = create_nft_list_table_sql(chain)?; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + conn.execute(&sql_nft_list, NO_PARAMS).map(|_| ())?; + Ok(()) + }) + .await + } + + async fn is_initialized(&self, chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let nft_list_initialized = query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [table_name], string_from_row)?; + Ok(nft_list_initialized.is_some()) + }) + .await + } async fn get_tx_history(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { todo!() From 64fa485c88b3ae7df4f2b4eb660826356c3cb7af Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 28 Apr 2023 11:33:57 +0700 Subject: [PATCH 022/112] save state --- mm2src/coins/eth.rs | 3 + mm2src/coins/nft.rs | 21 ++++-- mm2src/coins/nft/nft_errors.rs | 24 +++++++ mm2src/coins/nft/nft_structs.rs | 22 ++++++- mm2src/coins/nft_storage/mod.rs | 10 ++- mm2src/coins/nft_storage/sql_storage.rs | 65 +++++++++++++++++-- mm2src/db_common/src/sql_query.rs | 6 ++ .../mm2_main/src/rpc/dispatcher/dispatcher.rs | 6 +- 8 files changed, 141 insertions(+), 16 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index f610b91845..a79c9fa874 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -879,6 +879,9 @@ pub async fn withdraw_erc1155(ctx: MmArc, req: WithdrawErc1155) -> WithdrawNftRe // dont use `get_nft_metadata` for erc1155, it can return info related to other owner. let nft_req = NftListReq { chains: vec![req.chain], + max: true, + limit: 0, + page_number: None, }; let wallet_amount = find_wallet_amount(ctx, nft_req, req.token_address.clone(), req.token_id.clone()).await?; diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 66848297a7..fde8ce1e51 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -5,9 +5,10 @@ pub(crate) mod nft_errors; pub(crate) mod nft_structs; use crate::WithdrawError; -use nft_errors::GetNftInfoError; +use nft_errors::{GetNftInfoError, UpdateNftError}; use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, NftTransferHistoryWrapper, - NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, WithdrawNftReq}; + NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, UpdateNftReq, + WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft_storage::{NftListStorageOps, NftStorageBuilder}; @@ -29,13 +30,15 @@ pub type WithdrawNftResult = Result MmResult { let mut res_list = Vec::new(); let storage = NftStorageBuilder::new(&ctx).build()?; - for chain in req.chains { - if NftListStorageOps::is_initialized(&storage, &chain).await? { - NftListStorageOps::init(&storage, &chain).await?; + for chain in req.chains.iter() { + if NftListStorageOps::is_initialized(&storage, chain).await? { + NftListStorageOps::init(&storage, chain).await?; } - let nfts = storage.get_nft_list(&ctx, &chain).await?; - res_list.extend(nfts); } + let nfts = storage + .get_nft_list(&ctx, req.chains, req.max, req.limit, req.page_number) + .await?; + res_list.extend(nfts); drop_mutability!(res_list); let nft_list = NftList { nfts: res_list }; Ok(nft_list) @@ -160,6 +163,10 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult MmResult<(), UpdateNftError> { todo!() } + +pub async fn refresh_nft_metadata(_ctx: MmArc, _req: UpdateNftReq) -> MmResult<(), UpdateNftError> { todo!() } + /// `withdraw_nft` function generates, signs and returns a transaction that transfers NFT /// from my address to recipient's address. /// This method generates a raw transaction which should then be broadcast using `send_raw_transaction`. diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 27d66fdfc1..e968fd6dd2 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -95,3 +95,27 @@ impl HttpStatusCode for GetNftInfoError { } } } + +#[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize, SerializeErrorType)] +#[serde(tag = "error_type", content = "error_data")] +pub enum UpdateNftError { + #[display(fmt = "DB error {}", _0)] + DbError(String), + #[display(fmt = "Internal: {}", _0)] + Internal(String), +} + +impl From for UpdateNftError { + fn from(err: T) -> Self { + let msg = format!("{:?}", err); + UpdateNftError::DbError(msg) + } +} + +impl HttpStatusCode for UpdateNftError { + fn status_code(&self) -> StatusCode { + match self { + UpdateNftError::DbError(_) | UpdateNftError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + } + } +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index b366981533..7cd5fa6321 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -1,12 +1,19 @@ use crate::{TransactionType, TxFeeDetails, WithdrawFee}; +use common::ten; use mm2_number::BigDecimal; use rpc::v1::types::Bytes as BytesJson; use serde::Deserialize; +use std::num::NonZeroUsize; use std::str::FromStr; #[derive(Debug, Deserialize)] pub struct NftListReq { pub(crate) chains: Vec, + #[serde(default)] + pub(crate) max: bool, + #[serde(default = "ten")] + pub(crate) limit: usize, + pub(crate) page_number: Option, } #[derive(Debug, Deserialize)] @@ -226,9 +233,16 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } +#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, + pub(crate) filters: Option, + #[serde(default)] + pub(crate) max: bool, + #[serde(default = "ten")] + pub(crate) limit: usize, + pub(crate) page_number: Option, } #[derive(Debug, Serialize)] @@ -283,7 +297,7 @@ pub struct NftsTransferHistoryList { #[allow(dead_code)] #[derive(Debug, Deserialize)] -pub struct GetNftTxHistoryFilters { +pub struct NftTxHistoryFilters { from_block: Option, to_block: Option, from_address: Option, @@ -291,3 +305,9 @@ pub struct GetNftTxHistoryFilters { from_date: Option, to_date: Option, } + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +pub struct UpdateNftReq { + chains: Vec, +} diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 9d457893a1..bfd0ee408d 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -9,6 +9,7 @@ use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_err_handle::mm_error::{NotEqual, NotMmError}; use serde::{Deserialize, Serialize}; use std::format; +use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; @@ -25,7 +26,14 @@ pub trait NftListStorageOps { /// Whether tables are initialized for the specified chain. async fn is_initialized(&self, chain: &Chain) -> MmResult; - async fn get_nft_list(&self, ctx: &MmArc, chain: &Chain) -> MmResult, Self::Error>; + async fn get_nft_list( + &self, + ctx: &MmArc, + chains: Vec, + max: bool, + limit: usize, + page_number: Option, + ) -> MmResult, Self::Error>; async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index f5c881e985..555a36510f 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -2,19 +2,20 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftTransferHistory}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; use common::async_blocking; +use db_common::sql_build::SqlQuery; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; +use mm2_err_handle::or_mm_error::OrMmError; +use std::convert::TryInto; +use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; -#[allow(dead_code)] fn nft_list_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } -#[allow(dead_code)] fn nft_tx_history_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_tx_history" } -#[allow(dead_code)] fn create_nft_list_table_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -88,6 +89,27 @@ impl SqliteNftStorage { } } +fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmResult { + let mut union_sql_strings = Vec::new(); + for chain in chains.iter() { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let sql_builder = nft_list_table_builder_preimage(conn, table_name.as_str())?; + let sql_string = sql_builder.sql()?; + union_sql_strings.push(sql_string); + } + let union_sql = union_sql_strings.join(" UNION ALL "); + let mut final_sql_builder = SqlQuery::select_from_alias(conn, &format!("({}) AS nft_list", union_sql), "nft_list")?; + + final_sql_builder.order_desc("nft_list.block_number")?; + Ok(final_sql_builder) +} + +fn nft_list_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> MmResult, SqlError> { + let sql_builder = SqlQuery::select_from(conn, table_name)?; + Ok(sql_builder) +} + #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -115,15 +137,46 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn get_nft_list(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_nft_list( + &self, + _ctx: &MmArc, + chains: Vec, + max: bool, + limit: usize, + page_number: Option, + ) -> MmResult, Self::Error> { + let res = Vec::new(); + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let sql_builder = get_nft_list_builder_preimage(&conn, chains)?; + let mut total_count_builder = sql_builder.clone(); + total_count_builder.count_all()?; + let total: isize = total_count_builder + .query_single_row(|row| row.get(0))? + .or_mm_err(|| SqlError::QueryReturnedNoRows)?; + let count_total = total.try_into().expect("count should be always above zero"); + + let (_offset, _limit) = if max { + (0, count_total) + } else { + match page_number { + Some(page) => ((page.get() - 1) * limit, limit), + None => (0, limit), + } + }; + + Ok(res) + }) + .await + } - async fn add_nfts_to_list(&self, chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { let selfi = self.clone(); - let _chain = *chain; async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let _sql_transaction = conn.transaction()?; diff --git a/mm2src/db_common/src/sql_query.rs b/mm2src/db_common/src/sql_query.rs index 4c7b6eb362..cf4158ede4 100644 --- a/mm2src/db_common/src/sql_query.rs +++ b/mm2src/db_common/src/sql_query.rs @@ -316,6 +316,12 @@ impl<'a> SqlQuery<'a> { .field(format!("ROW_NUMBER() OVER (ORDER BY {}) AS {}", order_by, alias)); Ok(self) } + + /// Count all rows + pub fn count_all(&mut self) -> SqlResult<&mut Self> { + self.sql_builder.count("*"); + Ok(self) + } } /// `SqlCondition` implements the following methods by default: diff --git a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs index 43ab6b8c9e..f1f2c9c3a0 100644 --- a/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs +++ b/mm2src/mm2_main/src/rpc/dispatcher/dispatcher.rs @@ -50,7 +50,7 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use mm2_rpc::mm_protocol::{MmRpcBuilder, MmRpcRequest, MmRpcVersion}; #[cfg(feature = "enable-nft-integration")] -use nft::{get_nft_list, get_nft_metadata, get_nft_transfers, withdraw_nft}; +use nft::{get_nft_list, get_nft_metadata, get_nft_transfers, refresh_nft_metadata, update_nft, withdraw_nft}; use serde::de::DeserializeOwned; use serde_json::{self as json, Value as Json}; use std::net::SocketAddr; @@ -185,6 +185,8 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, my_tx_history_v2_rpc).await, "orderbook" => handle_mmrpc(ctx, request, orderbook_rpc_v2).await, "recreate_swap_data" => handle_mmrpc(ctx, request, recreate_swap_data).await, + #[cfg(feature = "enable-nft-integration")] + "refresh_nft_metadata" => handle_mmrpc(ctx, request, refresh_nft_metadata).await, "remove_delegation" => handle_mmrpc(ctx, request, remove_delegation).await, "remove_node_from_version_stat" => handle_mmrpc(ctx, request, remove_node_from_version_stat).await, "sign_message" => handle_mmrpc(ctx, request, sign_message).await, @@ -194,6 +196,8 @@ async fn dispatcher_v2(request: MmRpcRequest, ctx: MmArc) -> DispatcherResult handle_mmrpc(ctx, request, stop_version_stat_collection).await, "trade_preimage" => handle_mmrpc(ctx, request, trade_preimage_rpc).await, "trezor_connection_status" => handle_mmrpc(ctx, request, trezor_connection_status).await, + #[cfg(feature = "enable-nft-integration")] + "update_nft" => handle_mmrpc(ctx, request, update_nft).await, "update_version_stat_collection" => handle_mmrpc(ctx, request, update_version_stat_collection).await, "verify_message" => handle_mmrpc(ctx, request, verify_message).await, "withdraw" => handle_mmrpc(ctx, request, withdraw).await, From 844c9e5c6470f4c424f5481589bab9907b562195 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 28 Apr 2023 16:40:03 +0700 Subject: [PATCH 023/112] get_nft_list with storage --- mm2src/coins/nft.rs | 8 ++--- mm2src/coins/nft/nft_structs.rs | 4 ++- mm2src/coins/nft_storage/mod.rs | 4 +-- mm2src/coins/nft_storage/sql_storage.rs | 46 ++++++++++++++++++------- mm2src/db_common/src/sql_query.rs | 11 ++++++ 5 files changed, 52 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index fde8ce1e51..ebca3555cb 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -28,20 +28,16 @@ pub type WithdrawNftResult = Result MmResult { - let mut res_list = Vec::new(); let storage = NftStorageBuilder::new(&ctx).build()?; for chain in req.chains.iter() { - if NftListStorageOps::is_initialized(&storage, chain).await? { + if !NftListStorageOps::is_initialized(&storage, chain).await? { NftListStorageOps::init(&storage, chain).await?; } } let nfts = storage .get_nft_list(&ctx, req.chains, req.max, req.limit, req.page_number) .await?; - res_list.extend(nfts); - drop_mutability!(res_list); - let nft_list = NftList { nfts: res_list }; - Ok(nft_list) + Ok(nfts) } /// `get_nft_metadata` function returns info of one specific NFT. diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 7cd5fa6321..3dd086478f 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -107,7 +107,7 @@ impl FromStr for ContractType { } } -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct Nft { pub(crate) chain: Chain, pub(crate) token_address: String, @@ -176,6 +176,8 @@ impl std::ops::Deref for SerdeStringWrap { #[derive(Debug, Serialize)] pub struct NftList { pub(crate) nfts: Vec, + pub(crate) skipped: usize, + pub(crate) total: usize, } #[derive(Clone, Deserialize)] diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index bfd0ee408d..d52d160e75 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,6 +1,6 @@ use crate::eth::get_eth_address; use crate::nft::nft_errors::GetNftInfoError; -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftTransferHistory, NftWrapper}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftWrapper}; use crate::nft::{send_moralis_request, FORMAT_DECIMAL_MORALIS, URL_MORALIS}; use async_trait::async_trait; use derive_more::Display; @@ -33,7 +33,7 @@ pub trait NftListStorageOps { max: bool, limit: usize, page_number: Option, - ) -> MmResult, Self::Error>; + ) -> MmResult; async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 555a36510f..9debdaf0cc 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,13 +1,15 @@ -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftTransferHistory}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::SqlQuery; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, NO_PARAMS}; +use db_common::sqlite::rusqlite::types::Type; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_err_handle::or_mm_error::OrMmError; +use serde_json::{self as json}; use std::convert::TryInto; use std::num::NonZeroUsize; use std::sync::{Arc, Mutex}; @@ -95,12 +97,11 @@ fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmRes let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; let sql_builder = nft_list_table_builder_preimage(conn, table_name.as_str())?; - let sql_string = sql_builder.sql()?; + let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); union_sql_strings.push(sql_string); } let union_sql = union_sql_strings.join(" UNION ALL "); - let mut final_sql_builder = SqlQuery::select_from_alias(conn, &format!("({}) AS nft_list", union_sql), "nft_list")?; - + let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; final_sql_builder.order_desc("nft_list.block_number")?; Ok(final_sql_builder) } @@ -110,6 +111,20 @@ fn nft_list_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str Ok(sql_builder) } +fn finalize_get_nft_list_sql_builder( + sql_builder: &mut SqlQuery, + offset: usize, + limit: usize, +) -> MmResult<(), SqlError> { + sql_builder.offset(offset).limit(limit); + Ok(()) +} + +fn nft_from_row(row: &Row<'_>) -> Result { + let json_string: String = row.get(0)?; + json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) +} + #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -144,20 +159,21 @@ impl NftListStorageOps for SqliteNftStorage { max: bool, limit: usize, page_number: Option, - ) -> MmResult, Self::Error> { - let res = Vec::new(); + ) -> MmResult { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let sql_builder = get_nft_list_builder_preimage(&conn, chains)?; + let mut sql_builder = get_nft_list_builder_preimage(&conn, chains)?; let mut total_count_builder = sql_builder.clone(); total_count_builder.count_all()?; + // let str = total_count_builder.clone().sql()?; + // println!("total_count_builder: \n {} \n", str); let total: isize = total_count_builder .query_single_row(|row| row.get(0))? .or_mm_err(|| SqlError::QueryReturnedNoRows)?; - let count_total = total.try_into().expect("count should be always above zero"); + let count_total = total.try_into().expect("count should not be failed"); - let (_offset, _limit) = if max { + let (offset, limit) = if max { (0, count_total) } else { match page_number { @@ -165,8 +181,14 @@ impl NftListStorageOps for SqliteNftStorage { None => (0, limit), } }; - - Ok(res) + finalize_get_nft_list_sql_builder(&mut sql_builder, offset, limit)?; + let nfts = sql_builder.query(nft_from_row)?; + let result = NftList { + nfts, + skipped: offset, + total: count_total, + }; + Ok(result) }) .await } diff --git a/mm2src/db_common/src/sql_query.rs b/mm2src/db_common/src/sql_query.rs index cf4158ede4..381e25a243 100644 --- a/mm2src/db_common/src/sql_query.rs +++ b/mm2src/db_common/src/sql_query.rs @@ -322,6 +322,17 @@ impl<'a> SqlQuery<'a> { self.sql_builder.count("*"); Ok(self) } + + /// Select from union tables + pub fn select_from_union_alias(conn: &'a Connection, union_sql: &str, alias: &'static str) -> SqlResult { + validate_table_name(alias)?; + Ok(SqlQuery { + conn, + sql_builder: SqlBuilder::select_from(format!("({}) AS {}", union_sql, alias)), + params: SqlParamsBuilder::default(), + ordering: Vec::default(), + }) + } } /// `SqlCondition` implements the following methods by default: From d9c36a5c98daf1a9a89cdd9cade66ff4cc7fb066 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 28 Apr 2023 17:05:41 +0700 Subject: [PATCH 024/112] add get_nft method --- mm2src/coins/nft.rs | 2 +- mm2src/coins/nft_storage/mod.rs | 11 +++++++++-- mm2src/coins/nft_storage/sql_storage.rs | 22 ++++++++++++++++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index ebca3555cb..47f22850f8 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -35,7 +35,7 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult, max: bool, limit: usize, @@ -40,7 +40,14 @@ pub trait NftListStorageOps { I: IntoIterator + Send + 'static, I::IntoIter: Send; - async fn remove_nft_from_list(&self, nft: Nft) -> MmResult<(), Self::Error>; + async fn get_nft(&self, chain: &Chain, token_address: String, token_id: BigDecimal) -> MmResult<(), Self::Error>; + + async fn remove_nft_from_list( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult<(), Self::Error>; } #[async_trait] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 9debdaf0cc..94106126f2 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -9,6 +9,7 @@ use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_err_handle::or_mm_error::OrMmError; +use mm2_number::BigDecimal; use serde_json::{self as json}; use std::convert::TryInto; use std::num::NonZeroUsize; @@ -154,7 +155,6 @@ impl NftListStorageOps for SqliteNftStorage { async fn get_nft_list( &self, - _ctx: &MmArc, chains: Vec, max: bool, limit: usize, @@ -166,8 +166,6 @@ impl NftListStorageOps for SqliteNftStorage { let mut sql_builder = get_nft_list_builder_preimage(&conn, chains)?; let mut total_count_builder = sql_builder.clone(); total_count_builder.count_all()?; - // let str = total_count_builder.clone().sql()?; - // println!("total_count_builder: \n {} \n", str); let total: isize = total_count_builder .query_single_row(|row| row.get(0))? .or_mm_err(|| SqlError::QueryReturnedNoRows)?; @@ -208,7 +206,23 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } + async fn get_nft( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + ) -> MmResult<(), Self::Error> { + todo!() + } + + async fn remove_nft_from_list( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + ) -> MmResult<(), Self::Error> { + todo!() + } } #[async_trait] From 62ebf7905b01ce8c0481ccbdbc9f7d7160d10935 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 28 Apr 2023 19:16:02 +0700 Subject: [PATCH 025/112] optimize get_nft_list_builder_preimage --- mm2src/coins/nft_storage/sql_storage.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 94106126f2..76eb2ae9aa 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -93,26 +93,27 @@ impl SqliteNftStorage { } fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmResult { - let mut union_sql_strings = Vec::new(); - for chain in chains.iter() { + let union_sql_strings: MmResult, SqlError> = chains.iter().map(|chain| { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; - let sql_builder = nft_list_table_builder_preimage(conn, table_name.as_str())?; + let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); - union_sql_strings.push(sql_string); - } + Ok(sql_string) + }).collect(); + + let union_sql_strings = union_sql_strings?; let union_sql = union_sql_strings.join(" UNION ALL "); let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; final_sql_builder.order_desc("nft_list.block_number")?; Ok(final_sql_builder) } -fn nft_list_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> MmResult, SqlError> { +fn nft_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> MmResult, SqlError> { let sql_builder = SqlQuery::select_from(conn, table_name)?; Ok(sql_builder) } -fn finalize_get_nft_list_sql_builder( +fn finalize_nft_sql_builder( sql_builder: &mut SqlQuery, offset: usize, limit: usize, @@ -179,7 +180,7 @@ impl NftListStorageOps for SqliteNftStorage { None => (0, limit), } }; - finalize_get_nft_list_sql_builder(&mut sql_builder, offset, limit)?; + finalize_nft_sql_builder(&mut sql_builder, offset, limit)?; let nfts = sql_builder.query(nft_from_row)?; let result = NftList { nfts, From 9811680df1539a5522e60bffc76af33b1b177b34 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 28 Apr 2023 19:30:53 +0700 Subject: [PATCH 026/112] fix fmt --- mm2src/coins/nft_storage/sql_storage.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 76eb2ae9aa..7198412601 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -93,13 +93,16 @@ impl SqliteNftStorage { } fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmResult { - let union_sql_strings: MmResult, SqlError> = chains.iter().map(|chain| { - let table_name = nft_list_table_name(chain); - validate_table_name(&table_name)?; - let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; - let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); - Ok(sql_string) - }).collect(); + let union_sql_strings: MmResult, SqlError> = chains + .iter() + .map(|chain| { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; + let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); + Ok(sql_string) + }) + .collect(); let union_sql_strings = union_sql_strings?; let union_sql = union_sql_strings.join(" UNION ALL "); @@ -113,11 +116,7 @@ fn nft_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> Ok(sql_builder) } -fn finalize_nft_sql_builder( - sql_builder: &mut SqlQuery, - offset: usize, - limit: usize, -) -> MmResult<(), SqlError> { +fn finalize_nft_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { sql_builder.offset(offset).limit(limit); Ok(()) } From d652772503f920a0acca201487bb2a1e50c46989 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 30 Apr 2023 18:48:37 +0700 Subject: [PATCH 027/112] wip update_nft --- mm2src/coins/nft.rs | 170 +++++++++++++++++++++++- mm2src/coins/nft/nft_errors.rs | 19 ++- mm2src/coins/nft/nft_structs.rs | 4 +- mm2src/coins/nft_storage/mod.rs | 79 ++--------- mm2src/coins/nft_storage/sql_storage.rs | 13 +- 5 files changed, 212 insertions(+), 73 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 47f22850f8..5145e25235 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -11,7 +11,8 @@ use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHi WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft_storage::{NftListStorageOps, NftStorageBuilder}; +use crate::nft::nft_structs::ConvertChain; +use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::{APPLICATION_JSON, X_API_KEY}; use http::header::ACCEPT; use mm2_number::BigDecimal; @@ -155,11 +156,42 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult MmResult<(), UpdateNftError> { todo!() } +pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNftError> { + let storage = NftStorageBuilder::new(&ctx).build()?; + for chain in req.chains.iter() { + if !NftListStorageOps::is_initialized(&storage, chain).await? + && !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? + { + NftListStorageOps::init(&storage, chain).await?; + NftTxHistoryStorageOps::init(&storage, chain).await?; + let nft_list = get_moralis_nft_list(&ctx, chain).await?; + NftListStorageOps::add_nfts_to_list(&storage, chain, nft_list).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; + NftTxHistoryStorageOps::add_txs_to_history(&storage, chain, nft_transfers).await?; + } else if !NftListStorageOps::is_initialized(&storage, chain).await? + && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? + { + todo!() + } else if NftListStorageOps::is_initialized(&storage, chain).await? + && !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? + { + todo!() + } else if NftListStorageOps::is_initialized(&storage, chain).await? + && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? + { + todo!() + } else { + continue; + } + } + todo!() +} pub async fn refresh_nft_metadata(_ctx: MmArc, _req: UpdateNftReq) -> MmResult<(), UpdateNftError> { todo!() } @@ -245,3 +277,137 @@ pub(crate) async fn find_wallet_amount( })?; Ok(nft.amount) } + +async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, GetNftInfoError> { + let api_key = ctx.conf["api_key"] + .as_str() + .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; + + let mut res_list = Vec::new(); + + let (coin_str, chain_str) = chain.to_ticker_chain(); + let my_address = get_eth_address(ctx, &coin_str).await?; + let uri_without_cursor = format!( + "{}{}/nft?chain={}&{}", + URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS + ); + + // The cursor returned in the previous response (used for getting the next page). + let mut cursor = String::new(); + loop { + let uri = format!("{}{}", uri_without_cursor, cursor); + let response = send_moralis_request(uri.as_str(), api_key).await?; + if let Some(nfts_list) = response["result"].as_array() { + for nft_json in nfts_list { + let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; + let nft = Nft { + chain: *chain, + token_address: nft_wrapper.token_address, + token_id: nft_wrapper.token_id.0, + amount: nft_wrapper.amount.0, + owner_of: nft_wrapper.owner_of, + token_hash: nft_wrapper.token_hash, + block_number_minted: *nft_wrapper.block_number_minted, + block_number: *nft_wrapper.block_number, + contract_type: nft_wrapper.contract_type.map(|v| v.0), + name: nft_wrapper.name, + symbol: nft_wrapper.symbol, + token_uri: nft_wrapper.token_uri, + metadata: nft_wrapper.metadata, + last_token_uri_sync: nft_wrapper.last_token_uri_sync, + last_metadata_sync: nft_wrapper.last_metadata_sync, + minter_address: nft_wrapper.minter_address, + possible_spam: nft_wrapper.possible_spam, + }; + // collect NFTs from the page + res_list.push(nft); + } + // if cursor is not null, there are other NFTs on next page, + // and we need to send new request with cursor to get info from the next page. + if let Some(cursor_res) = response["cursor"].as_str() { + cursor = format!("{}{}", "&cursor=", cursor_res); + continue; + } else { + break; + } + } + } + + drop_mutability!(res_list); + Ok(res_list) +} + +async fn get_moralis_nft_transfers( + ctx: &MmArc, + chain: &Chain, + from_block: Option, +) -> MmResult, GetNftInfoError> { + let api_key = ctx.conf["api_key"] + .as_str() + .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; + + let mut res_list = Vec::new(); + let (coin_str, chain_str) = match chain { + Chain::Avalanche => ("AVAX", "avalanche"), + Chain::Bsc => ("BNB", "bsc"), + Chain::Eth => ("ETH", "eth"), + Chain::Fantom => ("FTM", "fantom"), + Chain::Polygon => ("MATIC", "polygon"), + }; + let my_address = get_eth_address(ctx, coin_str).await?; + let from_block = match from_block { + Some(block) => { + format!("&from_block={}", block) + }, + None => "".into(), + }; + let uri_without_cursor = format!( + "{}{}/nft/transfers?chain={}&{}&{}{}", + URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS, DIRECTION_BOTH_MORALIS, from_block + ); + + // The cursor returned in the previous response (used for getting the next page). + let mut cursor = String::new(); + loop { + let uri = format!("{}{}", uri_without_cursor, cursor); + let response = send_moralis_request(uri.as_str(), api_key).await?; + if let Some(transfer_list) = response["result"].as_array() { + for transfer in transfer_list { + let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; + let transfer_history = NftTransferHistory { + chain: *chain, + block_number: *transfer_wrapper.block_number, + block_timestamp: transfer_wrapper.block_timestamp, + block_hash: transfer_wrapper.block_hash, + transaction_hash: transfer_wrapper.transaction_hash, + transaction_index: transfer_wrapper.transaction_index, + log_index: transfer_wrapper.log_index, + value: transfer_wrapper.value.0, + contract_type: transfer_wrapper.contract_type.0, + transaction_type: transfer_wrapper.transaction_type, + token_address: transfer_wrapper.token_address, + token_id: transfer_wrapper.token_id.0, + from_address: transfer_wrapper.from_address, + to_address: transfer_wrapper.to_address, + amount: transfer_wrapper.amount.0, + verified: transfer_wrapper.verified, + operator: transfer_wrapper.operator, + possible_spam: transfer_wrapper.possible_spam, + }; + // collect NFTs transfers from the page + res_list.push(transfer_history); + } + // if the cursor is not null, there are other NFTs transfers on next page, + // and we need to send new request with cursor to get info from the next page. + if let Some(cursor_res) = response["cursor"].as_str() { + cursor = format!("{}{}", "&cursor=", cursor_res); + continue; + } else { + break; + } + } + } + + drop_mutability!(res_list); + Ok(res_list) +} diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index e968fd6dd2..5270b94380 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -81,6 +81,11 @@ impl From for GetNftInfoError { } } +impl From for UpdateNftError { + // expand UpdateNftError::GetNftInfoError + fn from(e: GetNftInfoError) -> Self { UpdateNftError::GetNftInfoError(e) } +} + impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { @@ -103,6 +108,15 @@ pub enum UpdateNftError { DbError(String), #[display(fmt = "Internal: {}", _0)] Internal(String), + GetNftInfoError(GetNftInfoError), +} + +impl From for UpdateNftError { + fn from(e: CreateNftStorageError) -> Self { + match e { + CreateNftStorageError::Internal(err) => UpdateNftError::Internal(err), + } + } } impl From for UpdateNftError { @@ -115,7 +129,10 @@ impl From for UpdateNftError { impl HttpStatusCode for UpdateNftError { fn status_code(&self) -> StatusCode { match self { - UpdateNftError::DbError(_) | UpdateNftError::Internal(_) => StatusCode::INTERNAL_SERVER_ERROR, + // expand UpdateNftError::GetNftInfoError + UpdateNftError::DbError(_) | UpdateNftError::Internal(_) | UpdateNftError::GetNftInfoError(_) => { + StatusCode::INTERNAL_SERVER_ERROR + }, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 3dd086478f..cf13993504 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -295,6 +295,8 @@ pub(crate) struct NftTransferHistoryWrapper { #[derive(Debug, Serialize)] pub struct NftsTransferHistoryList { pub(crate) transfer_history: Vec, + pub(crate) skipped: usize, + pub(crate) total: usize, } #[allow(dead_code)] @@ -311,5 +313,5 @@ pub struct NftTxHistoryFilters { #[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct UpdateNftReq { - chains: Vec, + pub(crate) chains: Vec, } diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 43c28bdd8b..b88397bcef 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,15 +1,11 @@ -use crate::eth::get_eth_address; -use crate::nft::nft_errors::GetNftInfoError; -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftWrapper}; -use crate::nft::{send_moralis_request, FORMAT_DECIMAL_MORALIS, URL_MORALIS}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; -use mm2_err_handle::mm_error::{MmError, MmResult}; +use mm2_err_handle::mm_error::MmResult; use mm2_err_handle::mm_error::{NotEqual, NotMmError}; use mm2_number::BigDecimal; use serde::{Deserialize, Serialize}; -use std::format; use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; @@ -60,12 +56,21 @@ pub trait NftTxHistoryStorageOps { /// Whether tables are initialized for the specified chain. async fn is_initialized(&self, chain: &Chain) -> MmResult; - async fn get_tx_history(&self, ctx: &MmArc, chain: &Chain) -> MmResult, Self::Error>; + async fn get_tx_history( + &self, + ctx: &MmArc, + chains: Vec, + max: bool, + limit: usize, + page_number: Option, + ) -> MmResult; async fn add_txs_to_history(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; + + async fn get_latest_block(&self, _chain: &Chain) -> MmResult; } #[derive(Debug, Deserialize, Display, Serialize)] @@ -91,63 +96,3 @@ impl<'a> NftStorageBuilder<'a> { sql_storage::SqliteNftStorage::new(self.ctx) } } - -#[allow(dead_code)] -async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, GetNftInfoError> { - let api_key = ctx.conf["api_key"] - .as_str() - .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - - let mut res_list = Vec::new(); - - let (coin_str, chain_str) = chain.to_ticker_chain(); - let my_address = get_eth_address(ctx, &coin_str).await?; - let uri_without_cursor = format!( - "{}{}/nft?chain={}&{}", - URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS - ); - - // The cursor returned in the previous response (used for getting the next page). - let mut cursor = String::new(); - loop { - let uri = format!("{}{}", uri_without_cursor, cursor); - let response = send_moralis_request(uri.as_str(), api_key).await?; - if let Some(nfts_list) = response["result"].as_array() { - for nft_json in nfts_list { - let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; - let nft = Nft { - chain: *chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, - block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.map(|v| v.0), - name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, - }; - // collect NFTs from the page - res_list.push(nft); - } - // if cursor is not null, there are other NFTs on next page, - // and we need to send new request with cursor to get info from the next page. - if let Some(cursor_res) = response["cursor"].as_str() { - cursor = format!("{}{}", "&cursor=", cursor_res); - continue; - } else { - break; - } - } - } - - drop_mutability!(res_list); - Ok(res_list) -} diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 7198412601..e67cc371e6 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; use common::async_blocking; @@ -252,7 +252,14 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn get_tx_history(&self, _ctx: &MmArc, _chain: &Chain) -> MmResult, Self::Error> { + async fn get_tx_history( + &self, + _ctx: &MmArc, + _chain: Vec, + _max: bool, + _limit: usize, + _page_number: Option, + ) -> MmResult { todo!() } @@ -263,4 +270,6 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { { todo!() } + + async fn get_latest_block(&self, _chain: &Chain) -> MmResult { todo!() } } From 8a64b4a4b222b4e1be92635347506351692e2f4e Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 1 May 2023 18:07:30 +0700 Subject: [PATCH 028/112] wip update_nft --- mm2src/coins/nft.rs | 40 +++++++++++--------- mm2src/coins/nft/nft_structs.rs | 42 ++++++++++++++++----- mm2src/coins/nft_storage/sql_storage.rs | 50 +++++++++++++++++++++++-- 3 files changed, 101 insertions(+), 31 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 5145e25235..c69c24fe7c 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -54,11 +54,11 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult "avalanche", - Chain::Bsc => "bsc", - Chain::Eth => "eth", - Chain::Fantom => "fantom", - Chain::Polygon => "polygon", + Chain::Avalanche => "AVALANCHE", + Chain::Bsc => "BSC", + Chain::Eth => "ETH", + Chain::Fantom => "FANTOM", + Chain::Polygon => "POLYGON", }; let uri = format!( "{}nft/{}/{}?chain={}&{}", @@ -99,11 +99,11 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult ("AVAX", "avalanche"), - Chain::Bsc => ("BNB", "bsc"), - Chain::Eth => ("ETH", "eth"), - Chain::Fantom => ("FTM", "fantom"), - Chain::Polygon => ("MATIC", "polygon"), + Chain::Avalanche => ("AVAX", "AVALANCHE"), + Chain::Bsc => ("BNB", "BSC"), + Chain::Eth => ("ETH", "ETH"), + Chain::Fantom => ("FTM", "FANTOM"), + Chain::Polygon => ("MATIC", "POLYGON"), }; let my_address = get_eth_address(&ctx, coin_str).await?; let uri_without_cursor = format!( @@ -177,20 +177,24 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft } else if !NftListStorageOps::is_initialized(&storage, chain).await? && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? { + NftListStorageOps::init(&storage, chain).await?; + let nft_list = get_moralis_nft_list(&ctx, chain).await?; + NftListStorageOps::add_nfts_to_list(&storage, chain, nft_list).await?; todo!() } else if NftListStorageOps::is_initialized(&storage, chain).await? && !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? { + NftTxHistoryStorageOps::init(&storage, chain).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; + NftTxHistoryStorageOps::add_txs_to_history(&storage, chain, nft_transfers).await?; todo!() } else if NftListStorageOps::is_initialized(&storage, chain).await? && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? { todo!() - } else { - continue; } } - todo!() + Ok(()) } pub async fn refresh_nft_metadata(_ctx: MmArc, _req: UpdateNftReq) -> MmResult<(), UpdateNftError> { todo!() } @@ -348,11 +352,11 @@ async fn get_moralis_nft_transfers( let mut res_list = Vec::new(); let (coin_str, chain_str) = match chain { - Chain::Avalanche => ("AVAX", "avalanche"), - Chain::Bsc => ("BNB", "bsc"), - Chain::Eth => ("ETH", "eth"), - Chain::Fantom => ("FTM", "fantom"), - Chain::Polygon => ("MATIC", "polygon"), + Chain::Avalanche => ("AVAX", "AVALANCHE"), + Chain::Bsc => ("BNB", "BSC"), + Chain::Eth => ("ETH", "ETH"), + Chain::Fantom => ("FTM", "FANTOM"), + Chain::Polygon => ("MATIC", "POLYGON"), }; let my_address = get_eth_address(ctx, coin_str).await?; let from_block = match from_block { diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index cf13993504..fef864e43e 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -3,6 +3,7 @@ use common::ten; use mm2_number::BigDecimal; use rpc::v1::types::Bytes as BytesJson; use serde::Deserialize; +use std::fmt; use std::num::NonZeroUsize; use std::str::FromStr; @@ -52,11 +53,23 @@ impl ConvertChain for Chain { fn to_ticker_chain(&self) -> (String, String) { match self { - Chain::Avalanche => ("AVAX".to_owned(), "avalanche".to_owned()), - Chain::Bsc => ("BNB".to_owned(), "bsc".to_owned()), - Chain::Eth => ("ETH".to_owned(), "eth".to_owned()), - Chain::Fantom => ("FTM".to_owned(), "fantom".to_owned()), - Chain::Polygon => ("MATIC".to_owned(), "polygon".to_owned()), + Chain::Avalanche => ("AVAX".to_owned(), "AVALANCHE".to_owned()), + Chain::Bsc => ("BNB".to_owned(), "BSC".to_owned()), + Chain::Eth => ("ETH".to_owned(), "ETH".to_owned()), + Chain::Fantom => ("FTM".to_owned(), "FANTOM".to_owned()), + Chain::Polygon => ("MATIC".to_owned(), "POLYGON".to_owned()), + } + } +} + +impl fmt::Display for Chain { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Chain::Avalanche => write!(f, "AVALANCHE"), + Chain::Bsc => write!(f, "BSC"), + Chain::Eth => write!(f, "ETH"), + Chain::Fantom => write!(f, "FANTOM"), + Chain::Polygon => write!(f, "POLYGON"), } } } @@ -72,11 +85,11 @@ impl FromStr for Chain { #[inline] fn from_str(s: &str) -> Result { match s { - "avalanche" => Ok(Chain::Avalanche), - "bsc" => Ok(Chain::Bsc), - "eth" => Ok(Chain::Eth), - "fantom" => Ok(Chain::Fantom), - "polygon" => Ok(Chain::Polygon), + "AVALANCHE" => Ok(Chain::Avalanche), + "BSC" => Ok(Chain::Bsc), + "ETH" => Ok(Chain::Eth), + "FANTOM" => Ok(Chain::Fantom), + "POLYGON" => Ok(Chain::Polygon), _ => Err(ParseChainTypeError::UnsupportedCainType), } } @@ -107,6 +120,15 @@ impl FromStr for ContractType { } } +impl fmt::Display for ContractType { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ContractType::Erc1155 => write!(f, "ERC1155"), + ContractType::Erc721 => write!(f, "ERC721"), + } + } +} + #[derive(Debug, Deserialize, Serialize)] pub struct Nft { pub(crate) chain: Chain, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index e67cc371e6..f8c97c2f53 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -5,7 +5,7 @@ use common::async_blocking; use db_common::sql_build::SqlQuery; use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; -use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; +use db_common::sqlite::{query_single_row, rusqlite, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_err_handle::or_mm_error::OrMmError; @@ -40,6 +40,7 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { last_token_uri_sync TEXT, last_metadata_sync TEXT, minter_address TEXT, + possible_spam INTEGER, PRIMARY KEY (token_address, token_id) );", table_name @@ -69,6 +70,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { to_address TEXT NOT NULL, amount VARCHAR(256) NOT NULL, verified INTEGER NOT NULL, + possible_spam INTEGER, operator TEXT );", table_name @@ -126,6 +128,40 @@ fn nft_from_row(row: &Row<'_>) -> Result { json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) } +fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + + let sql = format!( + "INSERT OR IGNORE INTO {} ( + chain, token_address, token_id, amount, owner_of, token_hash, + block_number_minted, block_number, contract_type, name, symbol, + token_uri, metadata, last_token_uri_sync, last_metadata_sync, minter_address, possible_spam + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17 + );", + table_name + ); + Ok(sql) +} + +fn insert_nft_tx_in_history_sql(chain: &Chain) -> MmResult { + let table_name = nft_tx_history_table_name(chain); + validate_table_name(&table_name)?; + + let sql = format!( + "INSERT OR IGNORE INTO {} ( + transaction_hash, chain, block_number, block_timestamp, block_hash, + transaction_index, log_index, value, contract_type, transaction_type, + token_address, token_id, from_address, to_address, amount, verified, operator, possible_spam + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18 + );", + table_name + ); + Ok(sql) +} + #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -191,16 +227,24 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { let selfi = self.clone(); + let chain = chain.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); - let _sql_transaction = conn.transaction()?; + let sql_transaction = conn.transaction()?; + + for nft in nfts { + let params = []; + + sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; + } + sql_transaction.commit()?; Ok(()) }) .await From 330af7a7f94d206119f1d2d4b99bb9bd9b80001b Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 1 May 2023 20:58:56 +0700 Subject: [PATCH 029/112] wip update_nft --- mm2src/coins/nft_storage/sql_storage.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index f8c97c2f53..31b577feb5 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -5,7 +5,7 @@ use common::async_blocking; use db_common::sql_build::SqlQuery; use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; -use db_common::sqlite::{query_single_row, rusqlite, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; +use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_err_handle::or_mm_error::OrMmError; @@ -145,6 +145,7 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { Ok(sql) } +#[allow(dead_code)] fn insert_nft_tx_in_history_sql(chain: &Chain) -> MmResult { let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; @@ -233,13 +234,17 @@ impl NftListStorageOps for SqliteNftStorage { I::IntoIter: Send, { let selfi = self.clone(); - let chain = chain.clone(); + let chain = *chain; async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; for nft in nfts { - let params = []; + let params = [ + nft.symbol, + nft.token_uri, + nft.metadata, + ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; } From c31baa0f7f341d81d7f644d51134b007aa6166b8 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 3 May 2023 18:27:49 +0700 Subject: [PATCH 030/112] add tx to history --- mm2src/coins/nft_storage/mod.rs | 4 +- mm2src/coins/nft_storage/sql_storage.rs | 72 ++++++++++++++++++++----- 2 files changed, 60 insertions(+), 16 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index b88397bcef..ff22a281c9 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -65,12 +65,12 @@ pub trait NftTxHistoryStorageOps { page_number: Option, ) -> MmResult; - async fn add_txs_to_history(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, chain: &Chain, txs: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; - async fn get_latest_block(&self, _chain: &Chain) -> MmResult; + async fn get_latest_block(&self, chain: &Chain) -> MmResult; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 31b577feb5..7a37639348 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -24,9 +24,9 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { validate_table_name(&table_name)?; let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( - chain TEXT NOT NULL, token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, + chain TEXT NOT NULL, amount VARCHAR(256) NOT NULL, owner_of TEXT NOT NULL, token_hash TEXT NOT NULL, @@ -48,7 +48,6 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { Ok(sql) } -#[allow(dead_code)] fn create_tx_history_table_sql(chain: &Chain) -> MmResult { let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; @@ -70,8 +69,8 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { to_address TEXT NOT NULL, amount VARCHAR(256) NOT NULL, verified INTEGER NOT NULL, - possible_spam INTEGER, - operator TEXT + operator TEXT, + possible_spam INTEGER );", table_name ); @@ -145,8 +144,7 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { Ok(sql) } -#[allow(dead_code)] -fn insert_nft_tx_in_history_sql(chain: &Chain) -> MmResult { +fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; @@ -154,7 +152,8 @@ fn insert_nft_tx_in_history_sql(chain: &Chain) -> MmResult { "INSERT OR IGNORE INTO {} ( transaction_hash, chain, block_number, block_timestamp, block_hash, transaction_index, log_index, value, contract_type, transaction_type, - token_address, token_id, from_address, to_address, amount, verified, operator, possible_spam + token_address, token_id, from_address, to_address, amount, verified, + operator, possible_spam ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18 );", @@ -241,14 +240,26 @@ impl NftListStorageOps for SqliteNftStorage { for nft in nfts { let params = [ + Some(nft.chain.to_string()), + Some(nft.token_address), + Some(nft.token_id.to_string()), + Some(nft.amount.to_string()), + Some(nft.owner_of), + Some(nft.token_hash), + Some(nft.block_number_minted.to_string()), + Some(nft.block_number.to_string()), + nft.contract_type.map(|ct| ct.to_string()), + nft.name, nft.symbol, nft.token_uri, nft.metadata, + nft.last_token_uri_sync, + nft.last_metadata_sync, + nft.minter_address, + nft.possible_spam.map(i32::from).map(|v| v.to_string()), ]; - sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; } - sql_transaction.commit()?; Ok(()) }) @@ -280,17 +291,17 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let sql_nft_list = create_nft_list_table_sql(chain)?; + let sql_tx_history = create_tx_history_table_sql(chain)?; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - conn.execute(&sql_nft_list, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_tx_history, NO_PARAMS).map(|_| ())?; Ok(()) }) .await } async fn is_initialized(&self, chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); + let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; let selfi = self.clone(); async_blocking(move || { @@ -312,12 +323,45 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { todo!() } - async fn add_txs_to_history(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, chain: &Chain, txs: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { - todo!() + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + + for tx in txs { + let params = [ + Some(tx.transaction_hash), + Some(tx.chain.to_string()), + Some(tx.block_number.to_string()), + Some(tx.block_timestamp), + Some(tx.block_hash), + Some(tx.transaction_index.to_string()), + Some(tx.log_index.to_string()), + Some(tx.value.to_string()), + Some(tx.contract_type.to_string()), + Some(tx.transaction_type.to_string()), + Some(tx.token_address), + Some(tx.token_id.to_string()), + Some(tx.from_address), + Some(tx.to_address), + Some(tx.amount.to_string()), + Some(tx.verified.to_string()), + tx.operator, + tx.possible_spam.map(i32::from).map(|v| v.to_string()), + ]; + + sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; + } + sql_transaction.commit()?; + Ok(()) + }) + .await } async fn get_latest_block(&self, _chain: &Chain) -> MmResult { todo!() } From 590cb76f034b542150f62e1619c94fd0bf2a13ce Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 4 May 2023 14:10:54 +0700 Subject: [PATCH 031/112] wip get table --- mm2src/coins/nft.rs | 79 ++++--------------------- mm2src/coins/nft/nft_structs.rs | 13 ++-- mm2src/coins/nft_storage/mod.rs | 4 +- mm2src/coins/nft_storage/sql_storage.rs | 46 ++++++++++++-- 4 files changed, 58 insertions(+), 84 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index c69c24fe7c..1f28e26101 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -83,7 +83,7 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult MmResult MmResult { - let api_key = ctx.conf["api_key"] - .as_str() - .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - - let mut res_list = Vec::new(); - - for chain in req.chains { - let (coin_str, chain_str) = match chain { - Chain::Avalanche => ("AVAX", "AVALANCHE"), - Chain::Bsc => ("BNB", "BSC"), - Chain::Eth => ("ETH", "ETH"), - Chain::Fantom => ("FTM", "FANTOM"), - Chain::Polygon => ("MATIC", "POLYGON"), - }; - let my_address = get_eth_address(&ctx, coin_str).await?; - let uri_without_cursor = format!( - "{}{}/nft/transfers?chain={}&{}&{}", - URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS, DIRECTION_BOTH_MORALIS - ); - - // The cursor returned in the previous response (used for getting the next page). - let mut cursor = String::new(); - loop { - let uri = format!("{}{}", uri_without_cursor, cursor); - let response = send_moralis_request(uri.as_str(), api_key).await?; - if let Some(transfer_list) = response["result"].as_array() { - for transfer in transfer_list { - let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; - let transfer_history = NftTransferHistory { - chain, - block_number: *transfer_wrapper.block_number, - block_timestamp: transfer_wrapper.block_timestamp, - block_hash: transfer_wrapper.block_hash, - transaction_hash: transfer_wrapper.transaction_hash, - transaction_index: transfer_wrapper.transaction_index, - log_index: transfer_wrapper.log_index, - value: transfer_wrapper.value.0, - contract_type: transfer_wrapper.contract_type.0, - transaction_type: transfer_wrapper.transaction_type, - token_address: transfer_wrapper.token_address, - token_id: transfer_wrapper.token_id.0, - from_address: transfer_wrapper.from_address, - to_address: transfer_wrapper.to_address, - amount: transfer_wrapper.amount.0, - verified: transfer_wrapper.verified, - operator: transfer_wrapper.operator, - possible_spam: transfer_wrapper.possible_spam, - }; - // collect NFTs transfers from the page - res_list.push(transfer_history); - } - // if the cursor is not null, there are other NFTs transfers on next page, - // and we need to send new request with cursor to get info from the next page. - if let Some(cursor_res) = response["cursor"].as_str() { - cursor = format!("{}{}", "&cursor=", cursor_res); - continue; - } else { - break; - } - } + let storage = NftStorageBuilder::new(&ctx).build()?; + for chain in req.chains.iter() { + if !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? { + NftTxHistoryStorageOps::init(&storage, chain).await?; } } - drop_mutability!(res_list); - let transfer_history_list = NftsTransferHistoryList { - transfer_history: res_list, - skipped: 0, - total: 0, - }; + let transfer_history_list = storage + .get_tx_history(req.chains, req.max, req.limit, req.page_number, req.filters) + .await?; Ok(transfer_history_list) } @@ -321,7 +262,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, last_token_uri_sync: nft_wrapper.last_token_uri_sync, last_metadata_sync: nft_wrapper.last_metadata_sync, minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, + possible_spam: nft_wrapper.possible_spam.map(i8::from), }; // collect NFTs from the page res_list.push(nft); @@ -396,7 +337,7 @@ async fn get_moralis_nft_transfers( amount: transfer_wrapper.amount.0, verified: transfer_wrapper.verified, operator: transfer_wrapper.operator, - possible_spam: transfer_wrapper.possible_spam, + possible_spam: transfer_wrapper.possible_spam.map(i8::from), }; // collect NFTs transfers from the page res_list.push(transfer_history); diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index fef864e43e..772528ef19 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -147,7 +147,7 @@ pub struct Nft { pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, - pub(crate) possible_spam: Option, + pub(crate) possible_spam: Option, } /// This structure is for deserializing NFT json to struct. @@ -257,7 +257,6 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, @@ -289,7 +288,7 @@ pub struct NftTransferHistory { pub(crate) amount: BigDecimal, pub(crate) verified: u64, pub(crate) operator: Option, - pub(crate) possible_spam: Option, + pub(crate) possible_spam: Option, } #[derive(Debug, Deserialize)] @@ -324,10 +323,10 @@ pub struct NftsTransferHistoryList { #[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTxHistoryFilters { - from_block: Option, - to_block: Option, - from_address: Option, - to_address: Option, + #[serde(default)] + receive: bool, + #[serde(default)] + send: bool, from_date: Option, to_date: Option, } diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index ff22a281c9..1787a24428 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftTxHistoryFilters, NftsTransferHistoryList}; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; @@ -58,11 +58,11 @@ pub trait NftTxHistoryStorageOps { async fn get_tx_history( &self, - ctx: &MmArc, chains: Vec, max: bool, limit: usize, page_number: Option, + filters: Option, ) -> MmResult; async fn add_txs_to_history(&self, chain: &Chain, txs: I) -> MmResult<(), Self::Error> diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 7a37639348..3d74807d12 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,4 +1,5 @@ -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftTxHistoryFilters, + NftsTransferHistoryList}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; use common::async_blocking; @@ -112,6 +113,30 @@ fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmRes Ok(final_sql_builder) } +// todo +fn get_nft_tx_builder_preimage( + conn: &Connection, + chains: Vec, + _filters: Option, +) -> MmResult { + let union_sql_strings: MmResult, SqlError> = chains + .iter() + .map(|chain| { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; + let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); + Ok(sql_string) + }) + .collect(); + + let union_sql_strings = union_sql_strings?; + let union_sql = union_sql_strings.join(" UNION ALL "); + let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; + final_sql_builder.order_desc("nft_list.block_number")?; + Ok(final_sql_builder) +} + fn nft_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> MmResult, SqlError> { let sql_builder = SqlQuery::select_from(conn, table_name)?; Ok(sql_builder) @@ -256,7 +281,7 @@ impl NftListStorageOps for SqliteNftStorage { nft.last_token_uri_sync, nft.last_metadata_sync, nft.minter_address, - nft.possible_spam.map(i32::from).map(|v| v.to_string()), + nft.possible_spam.map(|v| v.to_string()), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; } @@ -314,13 +339,22 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn get_tx_history( &self, - _ctx: &MmArc, - _chain: Vec, + chains: Vec, _max: bool, _limit: usize, _page_number: Option, + filters: Option, ) -> MmResult { - todo!() + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + // todo get_nft_tx_builder_preimage + let sql_builder = get_nft_tx_builder_preimage(&conn, chains, filters)?; + let mut total_count_builder = sql_builder.clone(); + total_count_builder.count_all()?; + todo!() + }) + .await } async fn add_txs_to_history(&self, chain: &Chain, txs: I) -> MmResult<(), Self::Error> @@ -353,7 +387,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.amount.to_string()), Some(tx.verified.to_string()), tx.operator, - tx.possible_spam.map(i32::from).map(|v| v.to_string()), + tx.possible_spam.map(|v| v.to_string()), ]; sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; From 8cc541361482d8ff4fd3b8fa7cdc2aeac1475586 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 4 May 2023 15:31:50 +0700 Subject: [PATCH 032/112] possible_spam bool --- mm2src/coins/nft.rs | 6 +++--- mm2src/coins/nft/nft_structs.rs | 4 ++-- mm2src/coins/nft_storage/sql_storage.rs | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 1f28e26101..33b4f497d1 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -83,7 +83,7 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult MmResult, last_token_uri_sync: nft_wrapper.last_token_uri_sync, last_metadata_sync: nft_wrapper.last_metadata_sync, minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam.map(i8::from), + possible_spam: nft_wrapper.possible_spam, }; // collect NFTs from the page res_list.push(nft); @@ -337,7 +337,7 @@ async fn get_moralis_nft_transfers( amount: transfer_wrapper.amount.0, verified: transfer_wrapper.verified, operator: transfer_wrapper.operator, - possible_spam: transfer_wrapper.possible_spam.map(i8::from), + possible_spam: transfer_wrapper.possible_spam, }; // collect NFTs transfers from the page res_list.push(transfer_history); diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 772528ef19..4f62bcd2c4 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -147,7 +147,7 @@ pub struct Nft { pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, - pub(crate) possible_spam: Option, + pub(crate) possible_spam: Option, } /// This structure is for deserializing NFT json to struct. @@ -288,7 +288,7 @@ pub struct NftTransferHistory { pub(crate) amount: BigDecimal, pub(crate) verified: u64, pub(crate) operator: Option, - pub(crate) possible_spam: Option, + pub(crate) possible_spam: Option, } #[derive(Debug, Deserialize)] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 3d74807d12..075b5cc5e6 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -281,7 +281,7 @@ impl NftListStorageOps for SqliteNftStorage { nft.last_token_uri_sync, nft.last_metadata_sync, nft.minter_address, - nft.possible_spam.map(|v| v.to_string()), + nft.possible_spam.map(i32::from).map(|v| v.to_string()), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; } @@ -387,7 +387,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.amount.to_string()), Some(tx.verified.to_string()), tx.operator, - tx.possible_spam.map(|v| v.to_string()), + tx.possible_spam.map(i32::from).map(|v| v.to_string()), ]; sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; From 9bae1afb464a86d4be744beefaf2cbcc9a9c3573 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 4 May 2023 19:11:18 +0700 Subject: [PATCH 033/112] fix get methods --- mm2src/coins/nft/nft_structs.rs | 2 +- mm2src/coins/nft_storage/sql_storage.rs | 79 ++++++++++++++++++------- 2 files changed, 59 insertions(+), 22 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 4f62bcd2c4..aedb6cf333 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -268,7 +268,7 @@ pub struct NftTransfersReq { pub(crate) page_number: Option, } -#[derive(Debug, Serialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct NftTransferHistory { pub(crate) chain: Chain, pub(crate) block_number: u64, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 075b5cc5e6..3b6e862bfe 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -42,6 +42,7 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { last_metadata_sync TEXT, minter_address TEXT, possible_spam INTEGER, + details_json TEXT, PRIMARY KEY (token_address, token_id) );", table_name @@ -71,7 +72,8 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { amount VARCHAR(256) NOT NULL, verified INTEGER NOT NULL, operator TEXT, - possible_spam INTEGER + possible_spam INTEGER, + details_json TEXT );", table_name ); @@ -108,12 +110,11 @@ fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmRes let union_sql_strings = union_sql_strings?; let union_sql = union_sql_strings.join(" UNION ALL "); - let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; - final_sql_builder.order_desc("nft_list.block_number")?; + let final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; Ok(final_sql_builder) } -// todo +// todo impl filters fn get_nft_tx_builder_preimage( conn: &Connection, chains: Vec, @@ -122,8 +123,9 @@ fn get_nft_tx_builder_preimage( let union_sql_strings: MmResult, SqlError> = chains .iter() .map(|chain| { - let table_name = nft_list_table_name(chain); + let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; + // todo here add filters let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); Ok(sql_string) @@ -132,8 +134,7 @@ fn get_nft_tx_builder_preimage( let union_sql_strings = union_sql_strings?; let union_sql = union_sql_strings.join(" UNION ALL "); - let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; - final_sql_builder.order_desc("nft_list.block_number")?; + let final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_history")?; Ok(final_sql_builder) } @@ -142,8 +143,16 @@ fn nft_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> Ok(sql_builder) } -fn finalize_nft_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { - sql_builder.offset(offset).limit(limit); +fn finalize_nft_list_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { + sql_builder.field("nft_list.details_json")?.offset(offset).limit(limit); + Ok(()) +} + +fn finalize_nft_history_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { + sql_builder + .field("nft_history.details_json")? + .offset(offset) + .limit(limit); Ok(()) } @@ -152,6 +161,11 @@ fn nft_from_row(row: &Row<'_>) -> Result { json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) } +fn tx_history_from_row(row: &Row<'_>) -> Result { + let json_string: String = row.get(0)?; + json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) +} + fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -160,9 +174,10 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { "INSERT OR IGNORE INTO {} ( chain, token_address, token_id, amount, owner_of, token_hash, block_number_minted, block_number, contract_type, name, symbol, - token_uri, metadata, last_token_uri_sync, last_metadata_sync, minter_address, possible_spam + token_uri, metadata, last_token_uri_sync, last_metadata_sync, minter_address, possible_spam, + details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18 );", table_name ); @@ -178,9 +193,9 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { transaction_hash, chain, block_number, block_timestamp, block_hash, transaction_index, log_index, value, contract_type, transaction_type, token_address, token_id, from_address, to_address, amount, verified, - operator, possible_spam + operator, possible_spam, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19 );", table_name ); @@ -240,7 +255,7 @@ impl NftListStorageOps for SqliteNftStorage { None => (0, limit), } }; - finalize_nft_sql_builder(&mut sql_builder, offset, limit)?; + finalize_nft_list_sql_builder(&mut sql_builder, offset, limit)?; let nfts = sql_builder.query(nft_from_row)?; let result = NftList { nfts, @@ -264,6 +279,7 @@ impl NftListStorageOps for SqliteNftStorage { let sql_transaction = conn.transaction()?; for nft in nfts { + let nft_json = json::to_string(&nft).expect("serialization should not fail"); let params = [ Some(nft.chain.to_string()), Some(nft.token_address), @@ -282,6 +298,7 @@ impl NftListStorageOps for SqliteNftStorage { nft.last_metadata_sync, nft.minter_address, nft.possible_spam.map(i32::from).map(|v| v.to_string()), + Some(nft_json), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; } @@ -340,19 +357,38 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn get_tx_history( &self, chains: Vec, - _max: bool, - _limit: usize, - _page_number: Option, + max: bool, + limit: usize, + page_number: Option, filters: Option, ) -> MmResult { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - // todo get_nft_tx_builder_preimage - let sql_builder = get_nft_tx_builder_preimage(&conn, chains, filters)?; + // todo get_nft_tx_builder_preimage complete filters + let mut sql_builder = get_nft_tx_builder_preimage(&conn, chains, filters)?; let mut total_count_builder = sql_builder.clone(); total_count_builder.count_all()?; - todo!() + let total: isize = total_count_builder + .query_single_row(|row| row.get(0))? + .or_mm_err(|| SqlError::QueryReturnedNoRows)?; + let count_total = total.try_into().expect("count should not be failed"); + let (offset, limit) = if max { + (0, count_total) + } else { + match page_number { + Some(page) => ((page.get() - 1) * limit, limit), + None => (0, limit), + } + }; + finalize_nft_history_sql_builder(&mut sql_builder, offset, limit)?; + let txs = sql_builder.query(tx_history_from_row)?; + let result = NftsTransferHistoryList { + transfer_history: txs, + skipped: offset, + total: count_total, + }; + Ok(result) }) .await } @@ -369,6 +405,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let sql_transaction = conn.transaction()?; for tx in txs { + let tx_json = json::to_string(&tx).expect("serialization should not fail"); let params = [ Some(tx.transaction_hash), Some(tx.chain.to_string()), @@ -388,8 +425,8 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.verified.to_string()), tx.operator, tx.possible_spam.map(i32::from).map(|v| v.to_string()), + Some(tx_json), ]; - sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; } sql_transaction.commit()?; From 1b10b14741a70a75f6734b3a9387fe5b341e1b50 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 5 May 2023 19:02:17 +0700 Subject: [PATCH 034/112] filters send and receive --- mm2src/coins/lp_coins.rs | 2 +- mm2src/coins/nft.rs | 14 ++++++-- mm2src/coins/nft/nft_errors.rs | 9 ++++- mm2src/coins/nft/nft_structs.rs | 10 +++--- mm2src/coins/nft_storage/mod.rs | 2 +- mm2src/coins/nft_storage/sql_storage.rs | 48 +++++++++++++++++++------ 6 files changed, 64 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 09eef3492b..0aef823c21 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -363,7 +363,7 @@ impl From for RawTransactionError { } } -#[derive(Debug, Deserialize, Display, EnumFromStringify, Serialize, SerializeErrorType)] +#[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize, SerializeErrorType)] #[serde(tag = "error_type", content = "error_data")] pub enum GetMyAddressError { CoinsConfCheckError(String), diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 33b4f497d1..02eb102cbb 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -4,7 +4,7 @@ use mm2_err_handle::prelude::{MmError, MmResult}; pub(crate) mod nft_errors; pub(crate) mod nft_structs; -use crate::WithdrawError; +use crate::{get_my_address, MyAddressReq, WithdrawError}; use nft_errors::{GetNftInfoError, UpdateNftError}; use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, NftTransferHistoryWrapper, NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, UpdateNftReq, @@ -97,8 +97,18 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult = req.chains.into_iter().zip(adds.into_iter()).collect(); let transfer_history_list = storage - .get_tx_history(req.chains, req.max, req.limit, req.page_number, req.filters) + .get_tx_history(chain_addr, req.max, req.limit, req.page_number, req.filters) .await?; Ok(transfer_history_list) } diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 5270b94380..216fb6ae5e 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,5 +1,6 @@ use crate::eth::GetEthAddressError; use crate::nft_storage::{CreateNftStorageError, NftStorageError}; +use crate::GetMyAddressError; use common::HttpStatusCode; use derive_more::Display; use enum_from::EnumFromStringify; @@ -36,6 +37,11 @@ pub enum GetNftInfoError { }, #[display(fmt = "DB error {}", _0)] DbError(String), + GetMyAddressError(GetMyAddressError), +} + +impl From for GetNftInfoError { + fn from(e: GetMyAddressError) -> Self { GetNftInfoError::GetMyAddressError(e) } } impl From for GetNftInfoError { @@ -96,7 +102,8 @@ impl HttpStatusCode for GetNftInfoError { | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) | GetNftInfoError::TokenNotFoundInWallet { .. } - | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::DbError(_) + | GetNftInfoError::GetMyAddressError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index aedb6cf333..bd4eac0086 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -321,14 +321,14 @@ pub struct NftsTransferHistoryList { } #[allow(dead_code)] -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] pub struct NftTxHistoryFilters { #[serde(default)] - receive: bool, + pub(crate) receive: bool, #[serde(default)] - send: bool, - from_date: Option, - to_date: Option, + pub(crate) send: bool, + pub(crate) from_date: Option, + pub(crate) to_date: Option, } #[allow(dead_code)] diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 1787a24428..95d272537f 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -58,7 +58,7 @@ pub trait NftTxHistoryStorageOps { async fn get_tx_history( &self, - chains: Vec, + chain_addr: Vec<(Chain, String)>, max: bool, limit: usize, page_number: Option, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 3b6e862bfe..96f6a221ee 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -3,7 +3,7 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHist use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use async_trait::async_trait; use common::async_blocking; -use db_common::sql_build::SqlQuery; +use db_common::sql_build::{SqlCondition, SqlQuery}; use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; @@ -114,20 +114,20 @@ fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmRes Ok(final_sql_builder) } -// todo impl filters fn get_nft_tx_builder_preimage( conn: &Connection, - chains: Vec, - _filters: Option, + chain_addr: Vec<(Chain, String)>, + filters: Option, ) -> MmResult { - let union_sql_strings: MmResult, SqlError> = chains - .iter() - .map(|chain| { - let table_name = nft_tx_history_table_name(chain); + let union_sql_strings: MmResult, SqlError> = chain_addr + .into_iter() + .map(|(chain, addr)| { + let table_name = nft_tx_history_table_name(&chain); validate_table_name(&table_name)?; // todo here add filters - let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; + let sql_builder = nft_history_table_builder_preimage(conn, table_name.as_str(), addr, filters.clone())?; let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); + println!("TABLE PREIMAGE = {} \n", sql_string); Ok(sql_string) }) .collect(); @@ -138,6 +138,30 @@ fn get_nft_tx_builder_preimage( Ok(final_sql_builder) } +fn nft_history_table_builder_preimage<'a>( + conn: &'a Connection, + table_name: &'a str, + owner_add: String, + filters: Option, +) -> Result, SqlError> { + let mut sql_builder = SqlQuery::select_from(conn, table_name)?; + if let Some(filters) = filters { + let owner_add = format!("'{}'", owner_add); + if filters.send && filters.receive { + sql_builder + .sql_builder() + .and_where_eq("from_address", owner_add.clone()) + .or_where_eq("to_address", owner_add); + } else if filters.send { + sql_builder.sql_builder().and_where_eq("from_address", owner_add); + } else if filters.receive { + sql_builder.sql_builder().and_where_eq("to_address", owner_add); + } + } + drop_mutability!(sql_builder); + Ok(sql_builder) +} + fn nft_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> MmResult, SqlError> { let sql_builder = SqlQuery::select_from(conn, table_name)?; Ok(sql_builder) @@ -356,7 +380,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn get_tx_history( &self, - chains: Vec, + chain_addr: Vec<(Chain, String)>, max: bool, limit: usize, page_number: Option, @@ -366,9 +390,11 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async_blocking(move || { let conn = selfi.0.lock().unwrap(); // todo get_nft_tx_builder_preimage complete filters - let mut sql_builder = get_nft_tx_builder_preimage(&conn, chains, filters)?; + let mut sql_builder = get_nft_tx_builder_preimage(&conn, chain_addr, filters)?; let mut total_count_builder = sql_builder.clone(); total_count_builder.count_all()?; + let str = total_count_builder.clone().sql()?; + println!("TOTAL COUNT = {} \n", str); let total: isize = total_count_builder .query_single_row(|row| row.get(0))? .or_mm_err(|| SqlError::QueryReturnedNoRows)?; From 12d0b602249bc3bbd3571bec22d2bc44820eb737 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 5 May 2023 22:34:42 +0700 Subject: [PATCH 035/112] from to filters in history --- mm2src/coins/nft/nft_structs.rs | 2 -- mm2src/coins/nft_storage/sql_storage.rs | 21 ++++++++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index bd4eac0086..b85e8b7300 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -320,7 +320,6 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } -#[allow(dead_code)] #[derive(Clone, Debug, Deserialize)] pub struct NftTxHistoryFilters { #[serde(default)] @@ -331,7 +330,6 @@ pub struct NftTxHistoryFilters { pub(crate) to_date: Option, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct UpdateNftReq { pub(crate) chains: Vec, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 96f6a221ee..39f08427a7 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -124,10 +124,8 @@ fn get_nft_tx_builder_preimage( .map(|(chain, addr)| { let table_name = nft_tx_history_table_name(&chain); validate_table_name(&table_name)?; - // todo here add filters let sql_builder = nft_history_table_builder_preimage(conn, table_name.as_str(), addr, filters.clone())?; let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); - println!("TABLE PREIMAGE = {} \n", sql_string); Ok(sql_string) }) .collect(); @@ -157,6 +155,22 @@ fn nft_history_table_builder_preimage<'a>( } else if filters.receive { sql_builder.sql_builder().and_where_eq("to_address", owner_add); } + + if filters.from_date.is_some() && filters.to_date.is_some() { + sql_builder.sql_builder().and_where(format!( + "block_timestamp BETWEEN '{}' and '{}'", + filters.from_date.unwrap(), + filters.to_date.unwrap() + )); + } else if filters.from_date.is_some() { + sql_builder + .sql_builder() + .and_where(format!("block_timestamp >= '{}'", filters.from_date.unwrap())); + } else if filters.to_date.is_some() { + sql_builder + .sql_builder() + .and_where(format!("block_timestamp <= '{}'", filters.to_date.unwrap())); + } } drop_mutability!(sql_builder); Ok(sql_builder) @@ -389,12 +403,9 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - // todo get_nft_tx_builder_preimage complete filters let mut sql_builder = get_nft_tx_builder_preimage(&conn, chain_addr, filters)?; let mut total_count_builder = sql_builder.clone(); total_count_builder.count_all()?; - let str = total_count_builder.clone().sql()?; - println!("TOTAL COUNT = {} \n", str); let total: isize = total_count_builder .query_single_row(|row| row.get(0))? .or_mm_err(|| SqlError::QueryReturnedNoRows)?; From 429161ab4922164f7968f2c00ea22b946de7b9fd Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 7 May 2023 17:44:56 +0700 Subject: [PATCH 036/112] get nft and latest block, delete nft --- mm2src/coins/nft_storage/mod.rs | 17 ++++- mm2src/coins/nft_storage/sql_storage.rs | 89 +++++++++++++++++++++---- 2 files changed, 91 insertions(+), 15 deletions(-) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 95d272537f..4b8735b214 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -11,6 +11,12 @@ use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm_storage; +#[derive(Debug)] +pub enum RemoveNftResult { + NftRemoved, + NftDidNotExist, +} + pub trait NftStorageError: std::fmt::Debug + NotMmError + NotEqual + Send {} #[async_trait] @@ -36,14 +42,19 @@ pub trait NftListStorageOps { I: IntoIterator + Send + 'static, I::IntoIter: Send; - async fn get_nft(&self, chain: &Chain, token_address: String, token_id: BigDecimal) -> MmResult<(), Self::Error>; + async fn get_nft( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult, Self::Error>; async fn remove_nft_from_list( &self, chain: &Chain, token_address: String, token_id: BigDecimal, - ) -> MmResult<(), Self::Error>; + ) -> MmResult; } #[async_trait] @@ -70,7 +81,7 @@ pub trait NftTxHistoryStorageOps { I: IntoIterator + Send + 'static, I::IntoIter: Send; - async fn get_latest_block(&self, chain: &Chain) -> MmResult; + async fn get_latest_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 39f08427a7..d77c15ab16 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,6 +1,7 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftTxHistoryFilters, NftsTransferHistoryList}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps, + RemoveNftResult}; use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; @@ -8,6 +9,7 @@ use db_common::sqlite::rusqlite::types::Type; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; +use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::mm_error::{MmError, MmResult}; use mm2_err_handle::or_mm_error::OrMmError; use mm2_number::BigDecimal; @@ -240,6 +242,38 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { Ok(sql) } +fn get_nft_metadata_sql(chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let sql = format!( + "SELECT details_json FROM {} WHERE token_address=?1 AND token_id=?2", + table_name + ); + Ok(sql) +} + +fn select_last_block_height_sql(chain: &Chain) -> MmResult { + let table_name = nft_list_table_name(chain); + validate_table_name(&table_name)?; + let sql = format!( + "SELECT block_number FROM {} ORDER BY block_number DESC LIMIT 1", + table_name + ); + Ok(sql) +} + +fn delete_nft_sql(chain: &Chain, table_name_creator: F) -> Result> +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + validate_table_name(&table_name)?; + let sql = format!("DELETE FROM {} WHERE token_address=?1 AND token_id=?2", table_name); + Ok(sql) +} + +fn block_height_from_row(row: &Row<'_>) -> Result { row.get(0) } + #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -348,20 +382,43 @@ impl NftListStorageOps for SqliteNftStorage { async fn get_nft( &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, - ) -> MmResult<(), Self::Error> { - todo!() + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult, Self::Error> { + let params = [token_address, token_id.to_string()]; + let sql = get_nft_metadata_sql(chain)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, params, nft_from_row).map_to_mm(SqlError::from) + }) + .await } async fn remove_nft_from_list( &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, - ) -> MmResult<(), Self::Error> { - todo!() + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult { + let sql = delete_nft_sql(chain, nft_list_table_name)?; + let params = [token_address, token_id.to_string()]; + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let rows_num = sql_transaction.execute(&sql, ¶ms)?; + + let remove_tx_result = if rows_num > 0 { + RemoveNftResult::NftRemoved + } else { + RemoveNftResult::NftDidNotExist + }; + sql_transaction.commit()?; + Ok(remove_tx_result) + }) + .await } } @@ -472,5 +529,13 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn get_latest_block(&self, _chain: &Chain) -> MmResult { todo!() } + async fn get_latest_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { + let sql = select_last_block_height_sql(chain)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, NO_PARAMS, block_height_from_row).map_to_mm(SqlError::from) + }) + .await + } } From d5efb2812474f4d62fc2ecb044f3e310c4ce1523 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 7 May 2023 18:23:44 +0700 Subject: [PATCH 037/112] get_nft_metadata --- mm2src/coins/nft.rs | 91 +++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 36 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 02eb102cbb..e1ba7fa57a 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -50,42 +50,21 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult { - let api_key = ctx.conf["api_key"] - .as_str() - .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - let chain_str = match req.chain { - Chain::Avalanche => "AVALANCHE", - Chain::Bsc => "BSC", - Chain::Eth => "ETH", - Chain::Fantom => "FANTOM", - Chain::Polygon => "POLYGON", - }; - let uri = format!( - "{}nft/{}/{}?chain={}&{}", - URL_MORALIS, req.token_address, req.token_id, chain_str, FORMAT_DECIMAL_MORALIS - ); - let response = send_moralis_request(uri.as_str(), api_key).await?; - let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string())?; - let nft_metadata = Nft { - chain: req.chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, - block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.map(|v| v.0), - name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, - }; - Ok(nft_metadata) + let storage = NftStorageBuilder::new(&ctx).build()?; + if !NftListStorageOps::is_initialized(&storage, &req.chain).await? { + NftListStorageOps::init(&storage, &req.chain).await?; + } + let nft = storage + .get_nft(&req.chain, req.token_address.clone(), req.token_id.clone()) + .await?; + if let Some(nft) = nft { + Ok(nft) + } else { + MmError::err(GetNftInfoError::TokenNotFoundInWallet { + token_address: req.token_address, + token_id: req.token_id.to_string(), + }) + } } /// `get_nft_transfers` function returns a transfer history of NFTs on requested chains owned by user. @@ -366,3 +345,43 @@ async fn get_moralis_nft_transfers( drop_mutability!(res_list); Ok(res_list) } + +#[allow(dead_code)] +async fn get_moralis_metadata(ctx: &MmArc, req: NftMetadataReq) -> MmResult { + let api_key = ctx.conf["api_key"] + .as_str() + .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; + let chain_str = match req.chain { + Chain::Avalanche => "AVALANCHE", + Chain::Bsc => "BSC", + Chain::Eth => "ETH", + Chain::Fantom => "FANTOM", + Chain::Polygon => "POLYGON", + }; + let uri = format!( + "{}nft/{}/{}?chain={}&{}", + URL_MORALIS, req.token_address, req.token_id, chain_str, FORMAT_DECIMAL_MORALIS + ); + let response = send_moralis_request(uri.as_str(), api_key).await?; + let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string())?; + let nft_metadata = Nft { + chain: req.chain, + token_address: nft_wrapper.token_address, + token_id: nft_wrapper.token_id.0, + amount: nft_wrapper.amount.0, + owner_of: nft_wrapper.owner_of, + token_hash: nft_wrapper.token_hash, + block_number_minted: *nft_wrapper.block_number_minted, + block_number: *nft_wrapper.block_number, + contract_type: nft_wrapper.contract_type.map(|v| v.0), + name: nft_wrapper.name, + symbol: nft_wrapper.symbol, + token_uri: nft_wrapper.token_uri, + metadata: nft_wrapper.metadata, + last_token_uri_sync: nft_wrapper.last_token_uri_sync, + last_metadata_sync: nft_wrapper.last_metadata_sync, + minter_address: nft_wrapper.minter_address, + possible_spam: nft_wrapper.possible_spam, + }; + Ok(nft_metadata) +} From e68155c7fbe1af8d56b188034fd5b452c4062b25 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 7 May 2023 22:39:34 +0700 Subject: [PATCH 038/112] refresh_nft_metadata --- mm2src/coins/nft.rs | 43 ++++++-- mm2src/coins/nft_storage/mod.rs | 9 ++ mm2src/coins/nft_storage/sql_storage.rs | 130 ++++++++++++++---------- 3 files changed, 117 insertions(+), 65 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index e1ba7fa57a..2269c0f12f 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -101,22 +101,22 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft NftListStorageOps::init(&storage, chain).await?; NftTxHistoryStorageOps::init(&storage, chain).await?; let nft_list = get_moralis_nft_list(&ctx, chain).await?; - NftListStorageOps::add_nfts_to_list(&storage, chain, nft_list).await?; + storage.add_nfts_to_list(chain, nft_list).await?; let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; - NftTxHistoryStorageOps::add_txs_to_history(&storage, chain, nft_transfers).await?; + storage.add_txs_to_history(chain, nft_transfers).await?; } else if !NftListStorageOps::is_initialized(&storage, chain).await? && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? { NftListStorageOps::init(&storage, chain).await?; let nft_list = get_moralis_nft_list(&ctx, chain).await?; - NftListStorageOps::add_nfts_to_list(&storage, chain, nft_list).await?; + storage.add_nfts_to_list(chain, nft_list).await?; todo!() } else if NftListStorageOps::is_initialized(&storage, chain).await? && !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? { NftTxHistoryStorageOps::init(&storage, chain).await?; let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; - NftTxHistoryStorageOps::add_txs_to_history(&storage, chain, nft_transfers).await?; + storage.add_txs_to_history(chain, nft_transfers).await?; todo!() } else if NftListStorageOps::is_initialized(&storage, chain).await? && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? @@ -127,7 +127,26 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft Ok(()) } -pub async fn refresh_nft_metadata(_ctx: MmArc, _req: UpdateNftReq) -> MmResult<(), UpdateNftError> { todo!() } +pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<(), UpdateNftError> { + let moralis_meta = get_moralis_metadata(&ctx, req.token_address.clone(), req.token_id.clone(), &req.chain).await?; + let storage = NftStorageBuilder::new(&ctx).build()?; + let req = NftMetadataReq { + token_address: req.token_address, + token_id: req.token_id, + chain: req.chain, + }; + let mut nft = get_nft_metadata(ctx, req).await?; + nft.name = moralis_meta.name; + nft.symbol = moralis_meta.symbol; + nft.token_uri = moralis_meta.token_uri; + nft.metadata = moralis_meta.metadata; + nft.last_token_uri_sync = moralis_meta.last_token_uri_sync; + nft.last_metadata_sync = moralis_meta.last_metadata_sync; + nft.possible_spam = moralis_meta.possible_spam; + drop_mutability!(nft); + storage.refresh_nft_metadata(&moralis_meta.chain, nft).await?; + Ok(()) +} /// `withdraw_nft` function generates, signs and returns a transaction that transfers NFT /// from my address to recipient's address. @@ -346,12 +365,16 @@ async fn get_moralis_nft_transfers( Ok(res_list) } -#[allow(dead_code)] -async fn get_moralis_metadata(ctx: &MmArc, req: NftMetadataReq) -> MmResult { +async fn get_moralis_metadata( + ctx: &MmArc, + token_address: String, + token_id: BigDecimal, + chain: &Chain, +) -> MmResult { let api_key = ctx.conf["api_key"] .as_str() .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - let chain_str = match req.chain { + let chain_str = match chain { Chain::Avalanche => "AVALANCHE", Chain::Bsc => "BSC", Chain::Eth => "ETH", @@ -360,12 +383,12 @@ async fn get_moralis_metadata(ctx: &MmArc, req: NftMetadataReq) -> MmResult MmResult; + + async fn get_nft_amount( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult, Self::Error>; + + async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; } #[async_trait] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index d77c15ab16..440c85270b 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -31,19 +31,8 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { token_id VARCHAR(256) NOT NULL, chain TEXT NOT NULL, amount VARCHAR(256) NOT NULL, - owner_of TEXT NOT NULL, - token_hash TEXT NOT NULL, - block_number_minted INTEGER NOT NULL, block_number INTEGER NOT NULL, contract_type TEXT, - name TEXT, - symbol TEXT, - token_uri TEXT, - metadata BLOB, - last_token_uri_sync TEXT, - last_metadata_sync TEXT, - minter_address TEXT, - possible_spam INTEGER, details_json TEXT, PRIMARY KEY (token_address, token_id) );", @@ -61,20 +50,12 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { chain TEXT NOT NULL, block_number INTEGER NOT NULL, block_timestamp TEXT NOT NULL, - block_hash TEXT NOT NULL, - transaction_index INTEGER NOT NULL, - log_index INTEGER NOT NULL, - value VARCHAR(256) NOT NULL, contract_type TEXT NOT NULL, - transaction_type TEXT NOT NULL, token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, from_address TEXT NOT NULL, to_address TEXT NOT NULL, amount VARCHAR(256) NOT NULL, - verified INTEGER NOT NULL, - operator TEXT, - possible_spam INTEGER, details_json TEXT );", table_name @@ -212,12 +193,9 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT OR IGNORE INTO {} ( - chain, token_address, token_id, amount, owner_of, token_hash, - block_number_minted, block_number, contract_type, name, symbol, - token_uri, metadata, last_token_uri_sync, last_metadata_sync, minter_address, possible_spam, - details_json + token_address, token_id, chain, amount, block_number, contract_type, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18 + ?1, ?2, ?3, ?4, ?5, ?6, ?7 );", table_name ); @@ -230,18 +208,30 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT OR IGNORE INTO {} ( - transaction_hash, chain, block_number, block_timestamp, block_hash, - transaction_index, log_index, value, contract_type, transaction_type, - token_address, token_id, from_address, to_address, amount, verified, - operator, possible_spam, details_json + transaction_hash, chain, block_number, block_timestamp, contract_type, + token_address, token_id, from_address, to_address, amount, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 );", table_name ); Ok(sql) } +fn insert_details_json_sql(chain: &Chain, table_name_creator: F) -> MmResult +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + + validate_table_name(&table_name)?; + let sql = format!( + "UPDATE {} SET details_json = ?3 WHERE token_address = ?1 AND token_id = ?2;", + table_name + ); + Ok(sql) +} + fn get_nft_metadata_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -252,8 +242,11 @@ fn get_nft_metadata_sql(chain: &Chain) -> MmResult { Ok(sql) } -fn select_last_block_height_sql(chain: &Chain) -> MmResult { - let table_name = nft_list_table_name(chain); +fn select_last_block_number_sql(chain: &Chain, table_name_creator: F) -> MmResult +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); validate_table_name(&table_name)?; let sql = format!( "SELECT block_number FROM {} ORDER BY block_number DESC LIMIT 1", @@ -262,6 +255,19 @@ fn select_last_block_height_sql(chain: &Chain) -> MmResult { Ok(sql) } +fn get_nft_amount_sql(chain: &Chain, table_name_creator: F) -> MmResult +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + validate_table_name(&table_name)?; + let sql = format!( + "SELECT amount FROM {} WHERE token_address=?1 AND token_id=?2", + table_name + ); + Ok(sql) +} + fn delete_nft_sql(chain: &Chain, table_name_creator: F) -> Result> where F: FnOnce(&Chain) -> String, @@ -272,7 +278,9 @@ where Ok(sql) } -fn block_height_from_row(row: &Row<'_>) -> Result { row.get(0) } +fn block_number_from_row(row: &Row<'_>) -> Result { row.get(0) } + +fn nft_amount_from_row(row: &Row<'_>) -> Result { row.get(0) } #[async_trait] impl NftListStorageOps for SqliteNftStorage { @@ -353,23 +361,12 @@ impl NftListStorageOps for SqliteNftStorage { for nft in nfts { let nft_json = json::to_string(&nft).expect("serialization should not fail"); let params = [ - Some(nft.chain.to_string()), Some(nft.token_address), Some(nft.token_id.to_string()), + Some(nft.chain.to_string()), Some(nft.amount.to_string()), - Some(nft.owner_of), - Some(nft.token_hash), - Some(nft.block_number_minted.to_string()), Some(nft.block_number.to_string()), nft.contract_type.map(|ct| ct.to_string()), - nft.name, - nft.symbol, - nft.token_uri, - nft.metadata, - nft.last_token_uri_sync, - nft.last_metadata_sync, - nft.minter_address, - nft.possible_spam.map(i32::from).map(|v| v.to_string()), Some(nft_json), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; @@ -386,8 +383,8 @@ impl NftListStorageOps for SqliteNftStorage { token_address: String, token_id: BigDecimal, ) -> MmResult, Self::Error> { - let params = [token_address, token_id.to_string()]; let sql = get_nft_metadata_sql(chain)?; + let params = [token_address, token_id.to_string()]; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -420,6 +417,37 @@ impl NftListStorageOps for SqliteNftStorage { }) .await } + + async fn get_nft_amount( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult, Self::Error> { + let sql = get_nft_amount_sql(chain, nft_list_table_name)?; + let params = [token_address, token_id.to_string()]; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, params, nft_amount_from_row).map_to_mm(SqlError::from) + }) + .await + } + + async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { + let sql = insert_details_json_sql(chain, nft_list_table_name)?; + let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [Some(nft.token_address), Some(nft.token_id.to_string()), Some(nft_json)]; + sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } } #[async_trait] @@ -505,20 +533,12 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.chain.to_string()), Some(tx.block_number.to_string()), Some(tx.block_timestamp), - Some(tx.block_hash), - Some(tx.transaction_index.to_string()), - Some(tx.log_index.to_string()), - Some(tx.value.to_string()), Some(tx.contract_type.to_string()), - Some(tx.transaction_type.to_string()), Some(tx.token_address), Some(tx.token_id.to_string()), Some(tx.from_address), Some(tx.to_address), Some(tx.amount.to_string()), - Some(tx.verified.to_string()), - tx.operator, - tx.possible_spam.map(i32::from).map(|v| v.to_string()), Some(tx_json), ]; sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; @@ -530,11 +550,11 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { } async fn get_latest_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { - let sql = select_last_block_height_sql(chain)?; + let sql = select_last_block_number_sql(chain, nft_list_table_name)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - query_single_row(&conn, &sql, NO_PARAMS, block_height_from_row).map_to_mm(SqlError::from) + query_single_row(&conn, &sql, NO_PARAMS, block_number_from_row).map_to_mm(SqlError::from) }) .await } From 106e18d122427ed5dccbb5d19912189cfb432b5a Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 8 May 2023 12:20:45 +0700 Subject: [PATCH 039/112] save update nft --- mm2src/coins/nft.rs | 151 +++++++++++++++++++----- mm2src/coins/nft/nft_errors.rs | 44 +++++-- mm2src/coins/nft_storage/mod.rs | 12 +- mm2src/coins/nft_storage/sql_storage.rs | 89 +++++++++++++- 4 files changed, 248 insertions(+), 48 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 2269c0f12f..d8657087ed 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -11,7 +11,7 @@ use nft_structs::{Chain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHi WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::ConvertChain; +use crate::nft::nft_structs::{ContractType, ConvertChain}; use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::{APPLICATION_JSON, X_API_KEY}; use http::header::ACCEPT; @@ -34,6 +34,8 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult MmResult { let storage = NftStorageBuilder::new(&ctx).build()?; if !NftListStorageOps::is_initialized(&storage, &req.chain).await? { @@ -68,7 +63,6 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult MmResult { let storage = NftStorageBuilder::new(&ctx).build()?; for chain in req.chains.iter() { @@ -95,33 +89,124 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult MmResult<(), UpdateNftError> { let storage = NftStorageBuilder::new(&ctx).build()?; for chain in req.chains.iter() { - if !NftListStorageOps::is_initialized(&storage, chain).await? - && !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? - { - NftListStorageOps::init(&storage, chain).await?; + let tx_history_initialized = NftTxHistoryStorageOps::is_initialized(&storage, chain).await?; + let list_initialized = NftListStorageOps::is_initialized(&storage, chain).await?; + + if !tx_history_initialized { NftTxHistoryStorageOps::init(&storage, chain).await?; - let nft_list = get_moralis_nft_list(&ctx, chain).await?; - storage.add_nfts_to_list(chain, nft_list).await?; let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; storage.add_txs_to_history(chain, nft_transfers).await?; - } else if !NftListStorageOps::is_initialized(&storage, chain).await? - && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? - { + } else { + let last_tx_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, last_tx_block.map(|b| b + 1)).await?; + storage.add_txs_to_history(chain, nft_transfers).await?; + } + + if !list_initialized { NftListStorageOps::init(&storage, chain).await?; let nft_list = get_moralis_nft_list(&ctx, chain).await?; storage.add_nfts_to_list(chain, nft_list).await?; - todo!() - } else if NftListStorageOps::is_initialized(&storage, chain).await? - && !NftTxHistoryStorageOps::is_initialized(&storage, chain).await? - { - NftTxHistoryStorageOps::init(&storage, chain).await?; - let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; - storage.add_txs_to_history(chain, nft_transfers).await?; - todo!() - } else if NftListStorageOps::is_initialized(&storage, chain).await? - && NftTxHistoryStorageOps::is_initialized(&storage, chain).await? - { - todo!() + } else { + let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; + // check if last block number exists + if let Some(last_block) = last_nft_block { + // try to update nft list info using updated tx info from transfer history table + let txs = storage.get_txs_from_block(chain, last_block + 1).await?; + for tx in txs.into_iter() { + let req = MyAddressReq { + coin: chain.to_ticker(), + }; + let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); + match (tx.from_address == my_address, tx.contract_type) { + (true, ContractType::Erc721) => { + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id) + .await?; + }, + (false, ContractType::Erc721) => { + let mut nft = get_moralis_metadata(&ctx, tx.token_address, tx.token_id, chain).await?; + // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later + // than History by Wallet update + nft.owner_of = my_address; + nft.block_number = tx.block_number; + drop_mutability!(nft); + storage.add_nfts_to_list(chain, [nft]).await?; + }, + (true, ContractType::Erc1155) => { + let nft_db = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await?; + // change amount or delete nft from the list + if let Some(mut nft_db) = nft_db { + match nft_db.amount.cmp(&tx.amount) { + std::cmp::Ordering::Equal => { + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id) + .await?; + }, + std::cmp::Ordering::Greater => { + nft_db.amount -= tx.amount; + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage.update_amount_block_number(chain, nft_db).await?; + }, + std::cmp::Ordering::Less => { + return MmError::err(UpdateNftError::InsufficientAmountInCache { + amount_list: nft_db.amount.to_string(), + amount_history: tx.amount.to_string(), + }); + }, + } + } else { + // if nft list table is not empty token must exist + return MmError::err(UpdateNftError::TokenNotFoundInWallet { + token_address: tx.token_address, + token_id: tx.token_id.to_string(), + }); + } + }, + (false, ContractType::Erc1155) => { + let nft_db = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await?; + // change amount or add nft to the list + if let Some(mut nft_db) = nft_db { + nft_db.amount += tx.amount; + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage.update_amount_block_number(chain, nft_db).await?; + } else { + let moralis_meta = + get_moralis_metadata(&ctx, tx.token_address, tx.token_id.clone(), chain).await?; + let nft = Nft { + chain: *chain, + token_address: moralis_meta.token_address, + token_id: moralis_meta.token_id, + amount: Default::default(), + owner_of: my_address, + token_hash: moralis_meta.token_hash, + block_number_minted: moralis_meta.block_number_minted, + block_number: tx.block_number, + contract_type: moralis_meta.contract_type, + name: moralis_meta.name, + symbol: moralis_meta.symbol, + token_uri: moralis_meta.token_uri, + metadata: moralis_meta.metadata, + last_token_uri_sync: moralis_meta.last_token_uri_sync, + last_metadata_sync: moralis_meta.last_metadata_sync, + minter_address: moralis_meta.minter_address, + possible_spam: moralis_meta.possible_spam, + }; + storage.add_nfts_to_list(chain, [nft]).await?; + } + }, + } + } + } else { + // if nft list table is empty, we can try to get info from moralis + let nft_list = get_moralis_nft_list(&ctx, chain).await?; + storage.add_nfts_to_list(chain, nft_list).await?; + } } } Ok(()) @@ -214,6 +299,7 @@ async fn send_moralis_request(uri: &str, api_key: &str) -> MmResult MmResult, async fn get_moralis_nft_transfers( ctx: &MmArc, chain: &Chain, - from_block: Option, + from_block: Option, ) -> MmResult, GetNftInfoError> { let api_key = ctx.conf["api_key"] .as_str() @@ -365,6 +451,9 @@ async fn get_moralis_nft_transfers( Ok(res_list) } +/// **Caution:** ERC-1155 token can have a total supply more than 1, which means there could be several owners +/// of the same token. `get_nft_metadata` returns NFTs info with the most recent owner. +/// **Dont** use this function to get specific info about owner address, amount etc, you will get info not related to my_address. async fn get_moralis_metadata( ctx: &MmArc, token_address: String, diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 216fb6ae5e..807e2f15d1 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -27,7 +27,7 @@ pub enum GetNftInfoError { #[display(fmt = "X-API-Key is missing")] ApiKeyError, #[display( - fmt = "Token: token_address {}, token_id {} was not find in wallet", + fmt = "Token: token_address {}, token_id {} was not found in wallet", token_address, token_id )] @@ -87,11 +87,6 @@ impl From for GetNftInfoError { } } -impl From for UpdateNftError { - // expand UpdateNftError::GetNftInfoError - fn from(e: GetNftInfoError) -> Self { UpdateNftError::GetNftInfoError(e) } -} - impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { @@ -116,6 +111,25 @@ pub enum UpdateNftError { #[display(fmt = "Internal: {}", _0)] Internal(String), GetNftInfoError(GetNftInfoError), + GetMyAddressError(GetMyAddressError), + #[display( + fmt = "Token: token_address {}, token_id {} was not found in wallet", + token_address, + token_id + )] + TokenNotFoundInWallet { + token_address: String, + token_id: String, + }, + #[display( + fmt = "Insufficient amount NFT token in the cache: amount in list table before transfer {}, transferred {}", + amount_list, + amount_history + )] + InsufficientAmountInCache { + amount_list: String, + amount_history: String, + }, } impl From for UpdateNftError { @@ -126,6 +140,14 @@ impl From for UpdateNftError { } } +impl From for UpdateNftError { + fn from(e: GetNftInfoError) -> Self { UpdateNftError::GetNftInfoError(e) } +} + +impl From for UpdateNftError { + fn from(e: GetMyAddressError) -> Self { UpdateNftError::GetMyAddressError(e) } +} + impl From for UpdateNftError { fn from(err: T) -> Self { let msg = format!("{:?}", err); @@ -136,10 +158,12 @@ impl From for UpdateNftError { impl HttpStatusCode for UpdateNftError { fn status_code(&self) -> StatusCode { match self { - // expand UpdateNftError::GetNftInfoError - UpdateNftError::DbError(_) | UpdateNftError::Internal(_) | UpdateNftError::GetNftInfoError(_) => { - StatusCode::INTERNAL_SERVER_ERROR - }, + UpdateNftError::DbError(_) + | UpdateNftError::Internal(_) + | UpdateNftError::GetNftInfoError(_) + | UpdateNftError::GetMyAddressError(_) + | UpdateNftError::TokenNotFoundInWallet { .. } + | UpdateNftError::InsufficientAmountInCache { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index f0897caec7..bc9e3e6566 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -64,6 +64,10 @@ pub trait NftListStorageOps { ) -> MmResult, Self::Error>; async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; + + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + + async fn update_amount_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; } #[async_trait] @@ -90,7 +94,13 @@ pub trait NftTxHistoryStorageOps { I: IntoIterator + Send + 'static, I::IntoIter: Send; - async fn get_latest_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + + async fn get_txs_from_block( + &self, + chain: &Chain, + block_number: u32, + ) -> MmResult, Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 440c85270b..57c5698575 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -218,7 +218,7 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { Ok(sql) } -fn insert_details_json_sql(chain: &Chain, table_name_creator: F) -> MmResult +fn update_details_json_sql(chain: &Chain, table_name_creator: F) -> MmResult where F: FnOnce(&Chain) -> String, { @@ -226,7 +226,21 @@ where validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET details_json = ?3 WHERE token_address = ?1 AND token_id = ?2;", + "UPDATE {} SET details_json = ?1 WHERE token_address = ?2 AND token_id = ?3;", + table_name + ); + Ok(sql) +} + +fn update_amount_block_number_sql(chain: &Chain, table_name_creator: F) -> MmResult +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + + validate_table_name(&table_name)?; + let sql = format!( + "UPDATE {} SET amount = ?1, block_number = ?2, details_json = ?3 WHERE token_address = ?4 AND token_id = ?5;", table_name ); Ok(sql) @@ -282,6 +296,22 @@ fn block_number_from_row(row: &Row<'_>) -> Result { row.get(0) } fn nft_amount_from_row(row: &Row<'_>) -> Result { row.get(0) } +fn get_txs_from_block_builder<'a>( + conn: &'a Connection, + chain: &'a Chain, + block_number: u32, +) -> MmResult, SqlError> { + let table_name = nft_tx_history_table_name(chain); + validate_table_name(table_name.as_str())?; + let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; + sql_builder + .sql_builder() + .and_where(format!("block_number > '{}'", block_number)) + .field("details_json"); + drop_mutability!(sql_builder); + Ok(sql_builder) +} + #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -435,13 +465,44 @@ impl NftListStorageOps for SqliteNftStorage { } async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { - let sql = insert_details_json_sql(chain, nft_list_table_name)?; + let sql = update_details_json_sql(chain, nft_list_table_name)?; + let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [Some(nft_json), Some(nft.token_address), Some(nft.token_id.to_string())]; + sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { + let sql = select_last_block_number_sql(chain, nft_list_table_name)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, NO_PARAMS, block_number_from_row).map_to_mm(SqlError::from) + }) + .await + } + + async fn update_amount_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { + let sql = update_amount_block_number_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - let params = [Some(nft.token_address), Some(nft.token_id.to_string()), Some(nft_json)]; + let params = [ + Some(nft.amount.to_string()), + Some(nft.block_number.to_string()), + Some(nft_json), + Some(nft.token_address), + Some(nft.token_id.to_string()), + ]; sql_transaction.execute(&sql, ¶ms)?; sql_transaction.commit()?; Ok(()) @@ -549,8 +610,8 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn get_latest_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { - let sql = select_last_block_number_sql(chain, nft_list_table_name)?; + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { + let sql = select_last_block_number_sql(chain, nft_tx_history_table_name)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); @@ -558,4 +619,20 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { }) .await } + + async fn get_txs_from_block( + &self, + chain: &Chain, + block_number: u32, + ) -> MmResult, Self::Error> { + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let sql_builder = get_txs_from_block_builder(&conn, &chain, block_number)?; + let txs = sql_builder.query(tx_history_from_row)?; + Ok(txs) + }) + .await + } } From 3d5dd43a6182026942170b48d4aa3f292aae053f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 9 May 2023 15:37:44 +0700 Subject: [PATCH 040/112] order by block_number ASC for get txs from block --- mm2src/coins/nft.rs | 2 +- mm2src/coins/nft_storage/mod.rs | 2 ++ mm2src/coins/nft_storage/sql_storage.rs | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d8657087ed..40171662cc 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -110,7 +110,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; // check if last block number exists if let Some(last_block) = last_nft_block { - // try to update nft list info using updated tx info from transfer history table + // try to update nft list info using updated tx info from transfer history table. let txs = storage.get_txs_from_block(chain, last_block + 1).await?; for tx in txs.into_iter() { let req = MyAddressReq { diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index bc9e3e6566..5e5c63d066 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -96,6 +96,8 @@ pub trait NftTxHistoryStorageOps { async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + /// [`NftTxHistoryStorageOps::get_txs_from_block`] function returns transfers sorted by + /// block_number in ascending order. It is needed to update the NFT LIST table correctly. async fn get_txs_from_block( &self, chain: &Chain, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 57c5698575..5f7fb9eda4 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -307,6 +307,7 @@ fn get_txs_from_block_builder<'a>( sql_builder .sql_builder() .and_where(format!("block_number > '{}'", block_number)) + .order_asc("block_number") .field("details_json"); drop_mutability!(sql_builder); Ok(sql_builder) From 6fc16cf1e28a93eb69be17390545f55774bea45e Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 9 May 2023 16:39:40 +0700 Subject: [PATCH 041/112] order_desc for nft_list.block_number and nft_history.block_timestamp in get nft list and get nft history reqs --- mm2src/coins/nft.rs | 2 -- mm2src/coins/nft_storage/sql_storage.rs | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 40171662cc..f75986deaa 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -34,8 +34,6 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult) -> MmRes let union_sql_strings = union_sql_strings?; let union_sql = union_sql_strings.join(" UNION ALL "); - let final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; + let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; + final_sql_builder.order_desc("nft_list.block_number")?; + drop_mutability!(final_sql_builder); Ok(final_sql_builder) } @@ -115,7 +117,9 @@ fn get_nft_tx_builder_preimage( let union_sql_strings = union_sql_strings?; let union_sql = union_sql_strings.join(" UNION ALL "); - let final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_history")?; + let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_history")?; + final_sql_builder.order_desc("nft_history.block_timestamp")?; + drop_mutability!(final_sql_builder); Ok(final_sql_builder) } From 2e70203f94e3cfd4e0c65c4d050fbda5a61e6d2f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 9 May 2023 16:50:33 +0700 Subject: [PATCH 042/112] impl last scanned block --- mm2src/coins/nft.rs | 207 +++++++++++++----------- mm2src/coins/nft/nft_structs.rs | 10 +- mm2src/coins/nft_storage/mod.rs | 12 +- mm2src/coins/nft_storage/sql_storage.rs | 77 ++++++++- 4 files changed, 198 insertions(+), 108 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index f75986deaa..d488b7c513 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -106,100 +106,15 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft storage.add_nfts_to_list(chain, nft_list).await?; } else { let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; - // check if last block number exists - if let Some(last_block) = last_nft_block { + let last_scanned_block = storage.get_last_scanned_block(chain).await?; + + // check if last scanned block number exists in last scanned block table + // if not try to check blocks existence in NFT LIST table. + if let Some(last_block) = last_scanned_block { // try to update nft list info using updated tx info from transfer history table. - let txs = storage.get_txs_from_block(chain, last_block + 1).await?; - for tx in txs.into_iter() { - let req = MyAddressReq { - coin: chain.to_ticker(), - }; - let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); - match (tx.from_address == my_address, tx.contract_type) { - (true, ContractType::Erc721) => { - storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id) - .await?; - }, - (false, ContractType::Erc721) => { - let mut nft = get_moralis_metadata(&ctx, tx.token_address, tx.token_id, chain).await?; - // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later - // than History by Wallet update - nft.owner_of = my_address; - nft.block_number = tx.block_number; - drop_mutability!(nft); - storage.add_nfts_to_list(chain, [nft]).await?; - }, - (true, ContractType::Erc1155) => { - let nft_db = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) - .await?; - // change amount or delete nft from the list - if let Some(mut nft_db) = nft_db { - match nft_db.amount.cmp(&tx.amount) { - std::cmp::Ordering::Equal => { - storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id) - .await?; - }, - std::cmp::Ordering::Greater => { - nft_db.amount -= tx.amount; - nft_db.block_number = tx.block_number; - drop_mutability!(nft_db); - storage.update_amount_block_number(chain, nft_db).await?; - }, - std::cmp::Ordering::Less => { - return MmError::err(UpdateNftError::InsufficientAmountInCache { - amount_list: nft_db.amount.to_string(), - amount_history: tx.amount.to_string(), - }); - }, - } - } else { - // if nft list table is not empty token must exist - return MmError::err(UpdateNftError::TokenNotFoundInWallet { - token_address: tx.token_address, - token_id: tx.token_id.to_string(), - }); - } - }, - (false, ContractType::Erc1155) => { - let nft_db = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) - .await?; - // change amount or add nft to the list - if let Some(mut nft_db) = nft_db { - nft_db.amount += tx.amount; - nft_db.block_number = tx.block_number; - drop_mutability!(nft_db); - storage.update_amount_block_number(chain, nft_db).await?; - } else { - let moralis_meta = - get_moralis_metadata(&ctx, tx.token_address, tx.token_id.clone(), chain).await?; - let nft = Nft { - chain: *chain, - token_address: moralis_meta.token_address, - token_id: moralis_meta.token_id, - amount: Default::default(), - owner_of: my_address, - token_hash: moralis_meta.token_hash, - block_number_minted: moralis_meta.block_number_minted, - block_number: tx.block_number, - contract_type: moralis_meta.contract_type, - name: moralis_meta.name, - symbol: moralis_meta.symbol, - token_uri: moralis_meta.token_uri, - metadata: moralis_meta.metadata, - last_token_uri_sync: moralis_meta.last_token_uri_sync, - last_metadata_sync: moralis_meta.last_metadata_sync, - minter_address: moralis_meta.minter_address, - possible_spam: moralis_meta.possible_spam, - }; - storage.add_nfts_to_list(chain, [nft]).await?; - } - }, - } - } + update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; + } else if let Some(last_block) = last_nft_block { + update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; } else { // if nft list table is empty, we can try to get info from moralis let nft_list = get_moralis_nft_list(&ctx, chain).await?; @@ -297,7 +212,7 @@ async fn send_moralis_request(uri: &str, api_key: &str) -> MmResult( + ctx: MmArc, + storage: &T, + chain: &Chain, + scan_from_block: u32, +) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTxHistoryStorageOps, +{ + let txs = storage.get_txs_from_block(chain, scan_from_block).await?; + for tx in txs.into_iter() { + let req = MyAddressReq { + coin: chain.to_ticker(), + }; + let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); + match (tx.from_address == my_address, tx.contract_type) { + (true, ContractType::Erc721) => { + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .await?; + }, + (false, ContractType::Erc721) => { + let mut nft = get_moralis_metadata(&ctx, tx.token_address, tx.token_id, chain).await?; + // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later + // than History by Wallet update + nft.owner_of = my_address; + nft.block_number = tx.block_number; + drop_mutability!(nft); + storage.add_nfts_to_list(chain, [nft]).await?; + }, + (true, ContractType::Erc1155) => { + let nft_db = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await?; + // change amount or delete nft from the list + if let Some(mut nft_db) = nft_db { + match nft_db.amount.cmp(&tx.amount) { + std::cmp::Ordering::Equal => { + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .await?; + }, + std::cmp::Ordering::Greater => { + nft_db.amount -= tx.amount; + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage.update_amount_block_number(chain, nft_db).await?; + }, + std::cmp::Ordering::Less => { + return MmError::err(UpdateNftError::InsufficientAmountInCache { + amount_list: nft_db.amount.to_string(), + amount_history: tx.amount.to_string(), + }); + }, + } + } else { + // token must exist in NFT LIST table + return MmError::err(UpdateNftError::TokenNotFoundInWallet { + token_address: tx.token_address, + token_id: tx.token_id.to_string(), + }); + } + }, + (false, ContractType::Erc1155) => { + let nft_db = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await?; + // change amount or add nft to the list + if let Some(mut nft_db) = nft_db { + nft_db.amount += tx.amount; + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage.update_amount_block_number(chain, nft_db).await?; + } else { + let moralis_meta = get_moralis_metadata(&ctx, tx.token_address, tx.token_id.clone(), chain).await?; + let nft = Nft { + chain: *chain, + token_address: moralis_meta.token_address, + token_id: moralis_meta.token_id, + amount: tx.amount, + owner_of: my_address, + token_hash: moralis_meta.token_hash, + block_number_minted: moralis_meta.block_number_minted, + block_number: tx.block_number, + contract_type: moralis_meta.contract_type, + name: moralis_meta.name, + symbol: moralis_meta.symbol, + token_uri: moralis_meta.token_uri, + metadata: moralis_meta.metadata, + last_token_uri_sync: moralis_meta.last_token_uri_sync, + last_metadata_sync: moralis_meta.last_metadata_sync, + minter_address: moralis_meta.minter_address, + possible_spam: moralis_meta.possible_spam, + }; + storage.add_nfts_to_list(chain, [nft]).await?; + } + }, + } + } + Ok(()) +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index b85e8b7300..ee7fbdb8b4 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -24,6 +24,11 @@ pub struct NftMetadataReq { pub(crate) chain: Chain, } +#[derive(Debug, Display)] +pub enum ParseChainTypeError { + UnsupportedCainType, +} + #[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "UPPERCASE")] pub enum Chain { @@ -74,11 +79,6 @@ impl fmt::Display for Chain { } } -#[derive(Debug, Display)] -pub enum ParseChainTypeError { - UnsupportedCainType, -} - impl FromStr for Chain { type Err = ParseChainTypeError; diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 5e5c63d066..92d7207cd6 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -54,6 +54,7 @@ pub trait NftListStorageOps { chain: &Chain, token_address: String, token_id: BigDecimal, + scanned_block: u64, ) -> MmResult; async fn get_nft_amount( @@ -65,8 +66,14 @@ pub trait NftListStorageOps { async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; + /// `get_last_block_number` function returns the height of last block in NFT LIST table async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + /// `get_last_scanned_block` function returns the height of last scanned block + /// when token was added or removed from MFT LIST table. + async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error>; + + /// `update_amount_block_number` function sets a new amount and block_number of a particular token in NFT LIST table async fn update_amount_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; } @@ -96,12 +103,13 @@ pub trait NftTxHistoryStorageOps { async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; - /// [`NftTxHistoryStorageOps::get_txs_from_block`] function returns transfers sorted by + /// `get_txs_from_block` function returns transfers sorted by /// block_number in ascending order. It is needed to update the NFT LIST table correctly. + /// Includes from_block number in ordering. async fn get_txs_from_block( &self, chain: &Chain, - block_number: u32, + from_block: u32, ) -> MmResult, Self::Error>; } diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index ca9b91f1c3..4472b25baf 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -22,6 +22,8 @@ fn nft_list_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_list fn nft_tx_history_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_tx_history" } +fn scanned_nft_blocks_table_name() -> String { "scanned_nft_blocks".to_string() } + fn create_nft_list_table_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -63,6 +65,19 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { Ok(sql) } +fn create_scanned_nft_blocks_sql() -> MmResult { + let table_name = scanned_nft_blocks_table_name(); + validate_table_name(&table_name)?; + let sql = format!( + "CREATE TABLE IF NOT EXISTS {} ( + chain TEXT PRIMARY KEY, + last_scanned_block INTEGER + );", + table_name + ); + Ok(sql) +} + impl NftStorageError for SqlError {} #[derive(Clone)] @@ -222,6 +237,12 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { Ok(sql) } +fn insert_chain_into_scanned_blocks_sql() -> MmResult { + let table_name = scanned_nft_blocks_table_name(); + validate_table_name(&table_name)?; + Ok(format!("INSERT OR IGNORE INTO {} (chain) VALUES (?1);", table_name)) +} + fn update_details_json_sql(chain: &Chain, table_name_creator: F) -> MmResult where F: FnOnce(&Chain) -> String, @@ -250,6 +271,13 @@ where Ok(sql) } +fn update_last_scanned_block_sql() -> MmResult { + let table_name = scanned_nft_blocks_table_name(); + validate_table_name(&table_name)?; + let sql = format!("UPDATE {} SET last_scanned_block = ?1 WHERE chain = ?2;", table_name); + Ok(sql) +} + fn get_nft_metadata_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -273,6 +301,13 @@ where Ok(sql) } +fn select_last_scanned_block_sql() -> MmResult { + let table_name = scanned_nft_blocks_table_name(); + validate_table_name(&table_name)?; + let sql = format!("SELECT last_scanned_block FROM {} WHERE chain=?1", table_name,); + Ok(sql) +} + fn get_nft_amount_sql(chain: &Chain, table_name_creator: F) -> MmResult where F: FnOnce(&Chain) -> String, @@ -303,14 +338,14 @@ fn nft_amount_from_row(row: &Row<'_>) -> Result { row.get(0) } fn get_txs_from_block_builder<'a>( conn: &'a Connection, chain: &'a Chain, - block_number: u32, + from_block: u32, ) -> MmResult, SqlError> { let table_name = nft_tx_history_table_name(chain); validate_table_name(table_name.as_str())?; let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; sql_builder .sql_builder() - .and_where(format!("block_number > '{}'", block_number)) + .and_where(format!("block_number >= '{}'", from_block)) .order_asc("block_number") .field("details_json"); drop_mutability!(sql_builder); @@ -324,9 +359,13 @@ impl NftListStorageOps for SqliteNftStorage { async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { let selfi = self.clone(); let sql_nft_list = create_nft_list_table_sql(chain)?; + let scanned_blocks_param = [chain.to_ticker()]; async_blocking(move || { let conn = selfi.0.lock().unwrap(); conn.execute(&sql_nft_list, NO_PARAMS).map(|_| ())?; + conn.execute(&create_scanned_nft_blocks_sql()?, NO_PARAMS).map(|_| ())?; + conn.execute(&insert_chain_into_scanned_blocks_sql()?, scanned_blocks_param) + .map(|_| ())?; Ok(()) }) .await @@ -339,7 +378,13 @@ impl NftListStorageOps for SqliteNftStorage { async_blocking(move || { let conn = selfi.0.lock().unwrap(); let nft_list_initialized = query_single_row(&conn, CHECK_TABLE_EXISTS_SQL, [table_name], string_from_row)?; - Ok(nft_list_initialized.is_some()) + let scanned_nft_blocks_initialized = query_single_row( + &conn, + CHECK_TABLE_EXISTS_SQL, + [scanned_nft_blocks_table_name()], + string_from_row, + )?; + Ok(nft_list_initialized.is_some() && scanned_nft_blocks_initialized.is_some()) }) .await } @@ -405,6 +450,8 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft_json), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; + let scanned_block_params = [nft.block_number.to_string(), nft.chain.to_ticker()]; + sql_transaction.execute(&update_last_scanned_block_sql()?, scanned_block_params)?; } sql_transaction.commit()?; Ok(()) @@ -433,22 +480,25 @@ impl NftListStorageOps for SqliteNftStorage { chain: &Chain, token_address: String, token_id: BigDecimal, + scanned_block: u64, ) -> MmResult { let sql = delete_nft_sql(chain, nft_list_table_name)?; let params = [token_address, token_id.to_string()]; + let scanned_block_params = [scanned_block.to_string(), chain.to_ticker()]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; let rows_num = sql_transaction.execute(&sql, ¶ms)?; - let remove_tx_result = if rows_num > 0 { + let remove_nft_result = if rows_num > 0 { RemoveNftResult::NftRemoved } else { RemoveNftResult::NftDidNotExist }; + sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; - Ok(remove_tx_result) + Ok(remove_nft_result) }) .await } @@ -494,9 +544,21 @@ impl NftListStorageOps for SqliteNftStorage { .await } + async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { + let sql = select_last_scanned_block_sql()?; + let params = [chain.to_ticker()]; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, params, block_number_from_row).map_to_mm(SqlError::from) + }) + .await + } + async fn update_amount_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { let sql = update_amount_block_number_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let scanned_block_params = [nft.block_number.to_string(), chain.to_ticker()]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); @@ -509,6 +571,7 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.token_id.to_string()), ]; sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) @@ -628,13 +691,13 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn get_txs_from_block( &self, chain: &Chain, - block_number: u32, + from_block: u32, ) -> MmResult, Self::Error> { let selfi = self.clone(); let chain = *chain; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let sql_builder = get_txs_from_block_builder(&conn, &chain, block_number)?; + let sql_builder = get_txs_from_block_builder(&conn, &chain, from_block)?; let txs = sql_builder.query(tx_history_from_row)?; Ok(txs) }) From c08d996f9792b061c76a01e816e5961926f7ec0e Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 15 May 2023 14:59:25 +0700 Subject: [PATCH 043/112] update only amount when some erc1155 tokens were sent or received --- mm2src/coins/nft.rs | 40 ++++++++++++++----------- mm2src/coins/nft_storage/mod.rs | 4 +-- mm2src/coins/nft_storage/sql_storage.rs | 13 ++++---- 3 files changed, 31 insertions(+), 26 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d488b7c513..8bb8ec358c 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -84,6 +84,7 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult MmResult<(), UpdateNftError> { let storage = NftStorageBuilder::new(&ctx).build()?; for chain in req.chains.iter() { @@ -105,20 +106,27 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let nft_list = get_moralis_nft_list(&ctx, chain).await?; storage.add_nfts_to_list(chain, nft_list).await?; } else { - let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; let last_scanned_block = storage.get_last_scanned_block(chain).await?; + let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; - // check if last scanned block number exists in last scanned block table - // if not try to check blocks existence in NFT LIST table. - if let Some(last_block) = last_scanned_block { - // try to update nft list info using updated tx info from transfer history table. - update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; - } else if let Some(last_block) = last_nft_block { - update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; - } else { - // if nft list table is empty, we can try to get info from moralis - let nft_list = get_moralis_nft_list(&ctx, chain).await?; - storage.add_nfts_to_list(chain, nft_list).await?; + match (last_scanned_block, last_nft_block) { + // Check if both block number exist, choose the highest for update_nft_list. + // Usually last_scanned_block should be the same + // or higher than last block in NFT LIST table (in the case of removing nft, then scanned block will be higher). + (Some(scanned_block), Some(nft_block)) => { + let last_block = std::cmp::max(scanned_block, nft_block); + update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; + }, + // check if last scanned block number exists in last scanned block table + // if not try to check blocks existence in NFT LIST table. + (Some(last_block), None) | (None, Some(last_block)) => { + update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; + }, + // if all blocks are empty we can try to get info from moralis + (None, None) => { + let nft_list = get_moralis_nft_list(&ctx, chain).await?; + storage.add_nfts_to_list(chain, nft_list).await?; + }, } } } @@ -412,7 +420,7 @@ async fn get_moralis_metadata( } /// `update_nft_list` function gets nft transfers from NFT HISTORY table, iterates through them -/// and updates NFT LIST table info +/// and updates NFT LIST table info. async fn update_nft_list( ctx: MmArc, storage: &T, @@ -457,9 +465,8 @@ where }, std::cmp::Ordering::Greater => { nft_db.amount -= tx.amount; - nft_db.block_number = tx.block_number; drop_mutability!(nft_db); - storage.update_amount_block_number(chain, nft_db).await?; + storage.update_nft_amount(chain, nft_db, tx.block_number).await?; }, std::cmp::Ordering::Less => { return MmError::err(UpdateNftError::InsufficientAmountInCache { @@ -483,9 +490,8 @@ where // change amount or add nft to the list if let Some(mut nft_db) = nft_db { nft_db.amount += tx.amount; - nft_db.block_number = tx.block_number; drop_mutability!(nft_db); - storage.update_amount_block_number(chain, nft_db).await?; + storage.update_nft_amount(chain, nft_db, tx.block_number).await?; } else { let moralis_meta = get_moralis_metadata(&ctx, tx.token_address, tx.token_id.clone(), chain).await?; let nft = Nft { diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 92d7207cd6..e5b8ce4d60 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -73,8 +73,8 @@ pub trait NftListStorageOps { /// when token was added or removed from MFT LIST table. async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error>; - /// `update_amount_block_number` function sets a new amount and block_number of a particular token in NFT LIST table - async fn update_amount_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; + /// `update_nft_amount` function sets a new amount of a particular token in NFT LIST table + async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error>; } #[async_trait] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 4472b25baf..15510619a4 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -71,7 +71,7 @@ fn create_scanned_nft_blocks_sql() -> MmResult { let sql = format!( "CREATE TABLE IF NOT EXISTS {} ( chain TEXT PRIMARY KEY, - last_scanned_block INTEGER + last_scanned_block INTEGER DEFAULT 0 );", table_name ); @@ -257,7 +257,7 @@ where Ok(sql) } -fn update_amount_block_number_sql(chain: &Chain, table_name_creator: F) -> MmResult +fn update_nft_amount_sql(chain: &Chain, table_name_creator: F) -> MmResult where F: FnOnce(&Chain) -> String, { @@ -265,7 +265,7 @@ where validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET amount = ?1, block_number = ?2, details_json = ?3 WHERE token_address = ?4 AND token_id = ?5;", + "UPDATE {} SET amount = ?1, details_json = ?2 WHERE token_address = ?3 AND token_id = ?4;", table_name ); Ok(sql) @@ -555,17 +555,16 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn update_amount_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { - let sql = update_amount_block_number_sql(chain, nft_list_table_name)?; + async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { + let sql = update_nft_amount_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); - let scanned_block_params = [nft.block_number.to_string(), chain.to_ticker()]; + let scanned_block_params = [scanned_block.to_string(), chain.to_ticker()]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; let params = [ Some(nft.amount.to_string()), - Some(nft.block_number.to_string()), Some(nft_json), Some(nft.token_address), Some(nft.token_id.to_string()), From 342491cb8965ba2998f0e3d56af9189de6a38381 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 16 May 2023 06:27:00 +0700 Subject: [PATCH 044/112] fix last_scanned_block logic --- mm2src/coins/nft.rs | 60 ++++++++++++++++++------- mm2src/coins/nft/nft_errors.rs | 20 ++++++++- mm2src/coins/nft_storage/mod.rs | 5 ++- mm2src/coins/nft_storage/sql_storage.rs | 53 +++++++++++++++++++--- 4 files changed, 112 insertions(+), 26 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 8bb8ec358c..9bd0f1092d 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -104,28 +104,42 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft if !list_initialized { NftListStorageOps::init(&storage, chain).await?; let nft_list = get_moralis_nft_list(&ctx, chain).await?; - storage.add_nfts_to_list(chain, nft_list).await?; + let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain) + .await? + .unwrap_or(0); + storage.add_nfts_to_list(chain, nft_list, last_scanned_block).await?; } else { let last_scanned_block = storage.get_last_scanned_block(chain).await?; let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; match (last_scanned_block, last_nft_block) { - // Check if both block number exist, choose the highest for update_nft_list. - // Usually last_scanned_block should be the same - // or higher than last block in NFT LIST table (in the case of removing nft, then scanned block will be higher). + // if both block numbers exist, last scanned block should be equal + // or higher than last block number from NFT LIST table. (Some(scanned_block), Some(nft_block)) => { - let last_block = std::cmp::max(scanned_block, nft_block); - update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; + if scanned_block >= nft_block { + update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1).await?; + } else { + return MmError::err(UpdateNftError::InvalidBlockOrder { + last_scanned_block: scanned_block.to_string(), + last_nft_block: nft_block.to_string(), + }); + } + }, + // If the last scanned block value is absent, we cannot accurately update the NFT cache. + // This is because a situation may occur where a user doesn't transfer all ERC-1155 tokens, + // resulting in the block number of NFT remaining unchanged. + (None, Some(nft_block)) => { + return MmError::err(UpdateNftError::LastScannedBlockNotFound { + last_nft_block: nft_block.to_string(), + }); }, - // check if last scanned block number exists in last scanned block table - // if not try to check blocks existence in NFT LIST table. - (Some(last_block), None) | (None, Some(last_block)) => { - update_nft_list(ctx.clone(), &storage, chain, last_block + 1).await?; + // if there are no rows in NFT LIST table or in both tables there are no rows + // we can try to get all info from moralis. + (Some(_), None) => { + cache_nfts_from_moralis(&ctx, &storage, chain).await?; }, - // if all blocks are empty we can try to get info from moralis (None, None) => { - let nft_list = get_moralis_nft_list(&ctx, chain).await?; - storage.add_nfts_to_list(chain, nft_list).await?; + cache_nfts_from_moralis(&ctx, &storage, chain).await?; }, } } @@ -133,6 +147,17 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft Ok(()) } +async fn cache_nfts_from_moralis(ctx: &MmArc, storage: &T, chain: &Chain) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTxHistoryStorageOps, +{ + let nft_list = get_moralis_nft_list(ctx, chain).await?; + let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(storage, chain) + .await? + .unwrap_or(0); + Ok(storage.add_nfts_to_list(chain, nft_list, last_scanned_block).await?) +} + pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<(), UpdateNftError> { let moralis_meta = get_moralis_metadata(&ctx, req.token_address.clone(), req.token_id.clone(), &req.chain).await?; let storage = NftStorageBuilder::new(&ctx).build()?; @@ -449,7 +474,7 @@ where nft.owner_of = my_address; nft.block_number = tx.block_number; drop_mutability!(nft); - storage.add_nfts_to_list(chain, [nft]).await?; + storage.add_nfts_to_list(chain, [nft], tx.block_number as u32).await?; }, (true, ContractType::Erc1155) => { let nft_db = storage @@ -487,11 +512,12 @@ where let nft_db = storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await?; - // change amount or add nft to the list + // change amount and block number or add nft to the list if let Some(mut nft_db) = nft_db { nft_db.amount += tx.amount; + nft_db.block_number = tx.block_number; drop_mutability!(nft_db); - storage.update_nft_amount(chain, nft_db, tx.block_number).await?; + storage.update_nft_amount_and_block_number(chain, nft_db).await?; } else { let moralis_meta = get_moralis_metadata(&ctx, tx.token_address, tx.token_id.clone(), chain).await?; let nft = Nft { @@ -513,7 +539,7 @@ where minter_address: moralis_meta.minter_address, possible_spam: moralis_meta.possible_spam, }; - storage.add_nfts_to_list(chain, [nft]).await?; + storage.add_nfts_to_list(chain, [nft], tx.block_number as u32).await?; } }, } diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 807e2f15d1..8492faca97 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -130,6 +130,22 @@ pub enum UpdateNftError { amount_list: String, amount_history: String, }, + #[display( + fmt = "Last scanned nft block {} should be >= last block number in nft table {}", + last_scanned_block, + last_nft_block + )] + InvalidBlockOrder { + last_scanned_block: String, + last_nft_block: String, + }, + #[display( + fmt = "Last scanned block not found, while the last NFT block exists: {}", + last_nft_block + )] + LastScannedBlockNotFound { + last_nft_block: String, + }, } impl From for UpdateNftError { @@ -163,7 +179,9 @@ impl HttpStatusCode for UpdateNftError { | UpdateNftError::GetNftInfoError(_) | UpdateNftError::GetMyAddressError(_) | UpdateNftError::TokenNotFoundInWallet { .. } - | UpdateNftError::InsufficientAmountInCache { .. } => StatusCode::INTERNAL_SERVER_ERROR, + | UpdateNftError::InsufficientAmountInCache { .. } + | UpdateNftError::InvalidBlockOrder { .. } + | UpdateNftError::LastScannedBlockNotFound { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index e5b8ce4d60..7abe127bd2 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -37,7 +37,7 @@ pub trait NftListStorageOps { page_number: Option, ) -> MmResult; - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u32) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; @@ -75,6 +75,8 @@ pub trait NftListStorageOps { /// `update_nft_amount` function sets a new amount of a particular token in NFT LIST table async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error>; + + async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; } #[async_trait] @@ -105,7 +107,6 @@ pub trait NftTxHistoryStorageOps { /// `get_txs_from_block` function returns transfers sorted by /// block_number in ascending order. It is needed to update the NFT LIST table correctly. - /// Includes from_block number in ordering. async fn get_txs_from_block( &self, chain: &Chain, diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 15510619a4..4fb327d452 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -271,6 +271,20 @@ where Ok(sql) } +fn update_nft_amount_and_block_number_sql(chain: &Chain, table_name_creator: F) -> MmResult +where + F: FnOnce(&Chain) -> String, +{ + let table_name = table_name_creator(chain); + + validate_table_name(&table_name)?; + let sql = format!( + "UPDATE {} SET amount = ?1, block_number = ?2, details_json = ?3 WHERE token_address = ?4 AND token_id = ?5;", + table_name + ); + Ok(sql) +} + fn update_last_scanned_block_sql() -> MmResult { let table_name = scanned_nft_blocks_table_name(); validate_table_name(&table_name)?; @@ -359,13 +373,10 @@ impl NftListStorageOps for SqliteNftStorage { async fn init(&self, chain: &Chain) -> MmResult<(), Self::Error> { let selfi = self.clone(); let sql_nft_list = create_nft_list_table_sql(chain)?; - let scanned_blocks_param = [chain.to_ticker()]; async_blocking(move || { let conn = selfi.0.lock().unwrap(); conn.execute(&sql_nft_list, NO_PARAMS).map(|_| ())?; conn.execute(&create_scanned_nft_blocks_sql()?, NO_PARAMS).map(|_| ())?; - conn.execute(&insert_chain_into_scanned_blocks_sql()?, scanned_blocks_param) - .map(|_| ())?; Ok(()) }) .await @@ -427,7 +438,7 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u32) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -450,9 +461,10 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft_json), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; - let scanned_block_params = [nft.block_number.to_string(), nft.chain.to_ticker()]; - sql_transaction.execute(&update_last_scanned_block_sql()?, scanned_block_params)?; } + sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, [chain.to_ticker()])?; + let scanned_block_params = [last_scanned_block.to_string(), chain.to_ticker()]; + sql_transaction.execute(&update_last_scanned_block_sql()?, scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) @@ -484,6 +496,7 @@ impl NftListStorageOps for SqliteNftStorage { ) -> MmResult { let sql = delete_nft_sql(chain, nft_list_table_name)?; let params = [token_address, token_id.to_string()]; + let chain_param = [chain.to_ticker()]; let scanned_block_params = [scanned_block.to_string(), chain.to_ticker()]; let selfi = self.clone(); async_blocking(move || { @@ -496,6 +509,7 @@ impl NftListStorageOps for SqliteNftStorage { } else { RemoveNftResult::NftDidNotExist }; + sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, chain_param)?; sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; Ok(remove_nft_result) @@ -558,6 +572,7 @@ impl NftListStorageOps for SqliteNftStorage { async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { let sql = update_nft_amount_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let chain_param = [chain.to_ticker()]; let scanned_block_params = [scanned_block.to_string(), chain.to_ticker()]; let selfi = self.clone(); async_blocking(move || { @@ -570,6 +585,32 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.token_id.to_string()), ]; sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, chain_param)?; + sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { + let sql = update_nft_amount_and_block_number_sql(chain, nft_list_table_name)?; + let nft_json = json::to_string(&nft).expect("serialization should not fail"); + let chain_param = [chain.to_ticker()]; + let scanned_block_params = [nft.block_number.to_string(), chain.to_ticker()]; + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + let params = [ + Some(nft.amount.to_string()), + Some(nft.block_number.to_string()), + Some(nft_json), + Some(nft.token_address), + Some(nft.token_id.to_string()), + ]; + sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, chain_param)?; sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; Ok(()) From 158542cd2a91c7d5d61e64f4a253af45a09c58a3 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 21 May 2023 14:59:42 +0700 Subject: [PATCH 045/112] add uri meta, impl update collection name in txs history --- mm2src/coins/eth.rs | 24 +- mm2src/coins/lp_coins.rs | 6 +- mm2src/coins/nft.rs | 371 +++++++++++++---------- mm2src/coins/nft/nft_errors.rs | 20 +- mm2src/coins/nft/nft_structs.rs | 59 +++- mm2src/coins/nft/nft_tests.rs | 25 +- mm2src/coins/nft_storage/mod.rs | 25 +- mm2src/coins/nft_storage/sql_storage.rs | 143 +++++++-- mm2src/coins/nft_storage/wasm_storage.rs | 55 +++- 9 files changed, 483 insertions(+), 245 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 5a83c41f1f..3d46f00868 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -22,8 +22,7 @@ // use super::eth::Action::{Call, Create}; use crate::lp_price::get_base_price_in_rel; -use crate::nft::nft_structs::{ContractType, ConvertChain, NftListReq, TransactionNftDetails, WithdrawErc1155, - WithdrawErc721}; +use crate::nft::nft_structs::{ContractType, ConvertChain, TransactionNftDetails, WithdrawErc1155, WithdrawErc721}; use async_trait::async_trait; use bitcrypto::{keccak256, ripemd160, sha256}; use common::custom_futures::repeatable::{Ready, Retry, RetryOnError}; @@ -67,7 +66,6 @@ use std::ops::Deref; use std::str::FromStr; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::{Arc, Mutex}; -use url::Url; use web3::types::{Action as TraceAction, BlockId, BlockNumber, Bytes, CallRequest, FilterBuilder, Log, Trace, TraceFilterBuilder, Transaction as Web3Transaction, TransactionId, U64}; use web3::{self, Web3}; @@ -107,7 +105,7 @@ pub use rlp; mod web3_transport; #[path = "eth/v2_activation.rs"] pub mod v2_activation; -use crate::nft::{find_wallet_amount, WithdrawNftResult}; +use crate::nft::WithdrawNftResult; use v2_activation::{build_address_and_priv_key_policy, EthActivationV2Error}; mod nonce; @@ -875,28 +873,14 @@ async fn withdraw_impl(coin: EthCoin, req: WithdrawRequest) -> WithdrawResult { /// `withdraw_erc1155` function returns details of `ERC-1155` transaction including tx hex, /// which should be sent to`send_raw_transaction` RPC to broadcast the transaction. -pub async fn withdraw_erc1155(ctx: MmArc, withdraw_type: WithdrawErc1155, url: Url) -> WithdrawNftResult { +pub async fn withdraw_erc1155(ctx: MmArc, withdraw_type: WithdrawErc1155) -> WithdrawNftResult { let coin = lp_coinfind_or_err(&ctx, &withdraw_type.chain.to_ticker()).await?; let (to_addr, token_addr, eth_coin) = get_valid_nft_add_to_withdraw(coin, &withdraw_type.to, &withdraw_type.token_address)?; let my_address = eth_coin.my_address()?; // todo check amount in nft cache, instead of sending new moralis req - // dont use `get_nft_metadata` for erc1155, it can return info related to other owner. - let nft_req = NftListReq { - chains: vec![withdraw_type.chain], - url, - max: true, - limit: 0, - page_number: None, - }; - let wallet_amount = find_wallet_amount( - ctx, - nft_req, - withdraw_type.token_address.clone(), - withdraw_type.token_id.clone(), - ) - .await?; + let wallet_amount = BigDecimal::default(); let amount_dec = if withdraw_type.max { wallet_amount.clone() diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 7add4946cd..9fc38d400a 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -100,7 +100,6 @@ cfg_wasm32! { use mm2_db::indexed_db::{ConstructibleDb, DbLocked, SharedDb}; use tx_history_storage::wasm::{clear_tx_history, load_tx_history, save_tx_history, TxHistoryDb}; pub type TxHistoryDbLocked<'a> = DbLocked<'a, TxHistoryDb>; - #[cfg(feature = "enable-nft-integration")] use nft_storage::wasm_storage::NftCacheIDB; } @@ -289,7 +288,8 @@ use utxo::{BlockchainNetwork, GenerateTxError, UtxoFeeDetails, UtxoTx}; pub mod nft; use nft::nft_errors::GetNftInfoError; -#[cfg(feature = "enable-nft-integration")] pub mod nft_storage; +pub mod nft_storage; + #[cfg(not(target_arch = "wasm32"))] pub mod z_coin; #[cfg(not(target_arch = "wasm32"))] use z_coin::ZCoin; @@ -2552,7 +2552,6 @@ pub struct CoinsContext { tx_history_db: SharedDb, #[cfg(target_arch = "wasm32")] hd_wallet_db: SharedDb, - #[cfg(feature = "enable-nft-integration")] #[cfg(target_arch = "wasm32")] pub nft_cache_db: SharedDb, } @@ -2579,7 +2578,6 @@ impl CoinsContext { tx_history_db: ConstructibleDb::new(ctx).into_shared(), #[cfg(target_arch = "wasm32")] hd_wallet_db: ConstructibleDb::new_shared_db(ctx).into_shared(), - #[cfg(feature = "enable-nft-integration")] #[cfg(target_arch = "wasm32")] nft_cache_db: ConstructibleDb::new(ctx).into_shared(), }) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 15ce33e171..81bd3190b2 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -1,5 +1,6 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::{MmError, MmResult}; +use url::Url; pub(crate) mod nft_errors; pub(crate) mod nft_structs; @@ -7,16 +8,18 @@ pub(crate) mod nft_structs; use crate::{get_my_address, MyAddressReq, WithdrawError}; use nft_errors::{GetNftInfoError, UpdateNftError}; -use nft_structs::{Chain, ConvertChain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, NftTransferHistoryWrapper, - NftTransfersReq, NftWrapper, NftsTransferHistoryList, TransactionNftDetails, UpdateNftReq, - WithdrawNftReq, ContractType}; +use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, + NftTransferHistoryWrapper, NftTransfersReq, NftWrapper, NftsTransferHistoryList, + TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::WithdrawNftType; -use common::APPLICATION_JSON; +use crate::nft::nft_structs::{TransferStatus, UriMeta, WithdrawNftType}; use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; +use common::APPLICATION_JSON; +use enum_from::EnumFromStringify; use http::header::ACCEPT; use mm2_err_handle::map_to_mm::MapToMmResult; +use mm2_net::transport::SlurpError; use mm2_number::BigDecimal; use serde_json::Value as Json; @@ -27,6 +30,8 @@ const MORALIS_FORMAT_QUERY_VALUE: &str = "decimal"; /// query parameters for moralis request: The transfer direction const MORALIS_DIRECTION_QUERY_NAME: &str = "direction"; const MORALIS_DIRECTION_QUERY_VALUE: &str = "both"; +// The minimum block number from which to get the transfers +const MORALIS_FROM_BLOCK_QUERY_NAME: &str = "from_block"; pub type WithdrawNftResult = Result>; @@ -51,13 +56,13 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult MmResult = req.chains.into_iter().zip(adds.into_iter()).collect(); let transfer_history_list = storage - .get_tx_history(chain_addr, req.max, req.limit, req.page_number, req.filters) + .get_tx_history(req.chains, req.max, req.limit, req.page_number, req.filters) .await?; Ok(transfer_history_list) } @@ -96,21 +91,24 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft if !tx_history_initialized { NftTxHistoryStorageOps::init(&storage, chain).await?; - let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None, &req.url).await?; storage.add_txs_to_history(chain, nft_transfers).await?; } else { let last_tx_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain).await?; - let nft_transfers = get_moralis_nft_transfers(&ctx, chain, last_tx_block.map(|b| b + 1)).await?; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, last_tx_block.map(|b| b + 1), &req.url).await?; storage.add_txs_to_history(chain, nft_transfers).await?; } if !list_initialized { NftListStorageOps::init(&storage, chain).await?; - let nft_list = get_moralis_nft_list(&ctx, chain).await?; + let nft_list = get_moralis_nft_list(&ctx, chain, &req.url).await?; let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain) .await? .unwrap_or(0); - storage.add_nfts_to_list(chain, nft_list, last_scanned_block).await?; + storage + .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) + .await?; + update_coll_name_in_txs(&storage, chain, nft_list).await?; } else { let last_scanned_block = storage.get_last_scanned_block(chain).await?; let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; @@ -120,7 +118,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft // or higher than last block number from NFT LIST table. (Some(scanned_block), Some(nft_block)) => { if scanned_block >= nft_block { - update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1).await?; + update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; } else { return MmError::err(UpdateNftError::InvalidBlockOrder { last_scanned_block: scanned_block.to_string(), @@ -139,10 +137,12 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft // if there are no rows in NFT LIST table or in both tables there are no rows // we can try to get all info from moralis. (Some(_), None) => { - cache_nfts_from_moralis(&ctx, &storage, chain).await?; + let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; + update_coll_name_in_txs(&storage, chain, nfts).await?; }, (None, None) => { - cache_nfts_from_moralis(&ctx, &storage, chain).await?; + let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; + update_coll_name_in_txs(&storage, chain, nfts).await?; }, } } @@ -150,27 +150,54 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft Ok(()) } -async fn cache_nfts_from_moralis(ctx: &MmArc, storage: &T, chain: &Chain) -> MmResult<(), UpdateNftError> +async fn cache_nfts_from_moralis( + ctx: &MmArc, + storage: &T, + chain: &Chain, + url: &Url, +) -> MmResult, UpdateNftError> where T: NftListStorageOps + NftTxHistoryStorageOps, { - let nft_list = get_moralis_nft_list(ctx, chain).await?; + let nft_list = get_moralis_nft_list(ctx, chain, url).await?; let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(storage, chain) .await? .unwrap_or(0); - Ok(storage.add_nfts_to_list(chain, nft_list, last_scanned_block).await?) + storage + .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) + .await?; + Ok(nft_list) +} + +async fn update_coll_name_in_txs(storage: &T, chain: &Chain, nfts: Vec) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTxHistoryStorageOps, +{ + for nft in nfts.into_iter() { + storage + .update_txs_coll_name_by_token_addr_id(chain, nft.token_address, nft.token_id, nft.collection_name) + .await?; + } + Ok(()) } pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<(), UpdateNftError> { - let moralis_meta = get_moralis_metadata(&ctx, req.token_address.clone(), req.token_id.clone(), &req.chain).await?; + let moralis_meta = get_moralis_metadata( + format!("{:#02x}", req.token_address), + req.token_id.clone(), + &req.chain, + &req.url, + ) + .await?; let storage = NftStorageBuilder::new(&ctx).build()?; let req = NftMetadataReq { token_address: req.token_address, token_id: req.token_id, chain: req.chain, + url: req.url, }; let mut nft = get_nft_metadata(ctx, req).await?; - nft.name = moralis_meta.name; + nft.collection_name = moralis_meta.collection_name; nft.symbol = moralis_meta.symbol; nft.token_uri = moralis_meta.token_uri; nft.metadata = moralis_meta.metadata; @@ -182,112 +209,32 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<( Ok(()) } -/// `withdraw_nft` function generates, signs and returns a transaction that transfers NFT -/// from my address to recipient's address. -/// This method generates a raw transaction which should then be broadcast using `send_raw_transaction`. -pub async fn withdraw_nft(ctx: MmArc, req_type: WithdrawNftReq) -> WithdrawNftResult { - match req_type { - WithdrawNftReq::WithdrawErc1155(erc1155_req) => withdraw_erc1155(ctx, erc1155_req).await, - WithdrawNftReq::WithdrawErc721(erc721_req) => withdraw_erc721(ctx, erc721_req).await, - } -} - -#[cfg(not(target_arch = "wasm32"))] -pub async fn send_moralis_request(uri: &str, api_key: &str) -> MmResult { - use http::header::HeaderValue; - use mm2_net::transport::slurp_req_body; - - let request = http::Request::builder() - .method("GET") - .uri(uri) - .header(X_API_KEY, api_key) - .header(ACCEPT, HeaderValue::from_static(APPLICATION_JSON)) - .body(hyper::Body::from(""))?; - - let (status, _header, body) = slurp_req_body(request).await?; - if !status.is_success() { - return Err(MmError::new(GetNftInfoError::Transport(format!( - "Response !200 from {}: {}, {}", - uri, status, body - )))); - } - Ok(body) -} - -#[cfg(target_arch = "wasm32")] -async fn send_moralis_request(uri: &str, api_key: &str) -> MmResult { - use mm2_net::wasm_http::FetchRequest; - - macro_rules! try_or { - ($exp:expr, $errtype:ident) => { - match $exp { - Ok(x) => x, - Err(e) => return Err(MmError::new(GetNftInfoError::$errtype(ERRL!("{:?}", e)))), - } - }; - } - - let result = FetchRequest::get(uri) - .cors() - .body_utf8("".to_owned()) - .header(X_API_KEY, api_key) - .header(ACCEPT.as_str(), APPLICATION_JSON) - .request_str() - .await; - let (status_code, response_str) = try_or!(result, Transport); - if !status_code.is_success() { - return Err(MmError::new(GetNftInfoError::Transport(ERRL!( - "!200: {}, {}", - status_code, - response_str - )))); - } - - let response: Json = try_or!(serde_json::from_str(&response_str), InvalidResponse); - Ok(response) -} - -/// This function uses `get_nft_list` method to get the correct info about amount of specific NFT owned by my_address. -// todo it is used for withdrawing, remove it and use db instead -pub(crate) async fn find_wallet_amount( - ctx: MmArc, - nft_list: NftListReq, - token_address_req: String, - token_id_req: BigDecimal, -) -> MmResult { - let nft_list = get_nft_list(ctx, nft_list).await?.nfts; - let nft = nft_list - .into_iter() - .find(|nft| nft.token_address == token_address_req && nft.token_id == token_id_req) - .ok_or_else(|| GetNftInfoError::TokenNotFoundInWallet { - token_address: token_address_req, - token_id: token_id_req.to_string(), - })?; - Ok(nft.amount) -} - -async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, GetNftInfoError> { - let api_key = ctx.conf["api_key"] - .as_str() - .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - +async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult, GetNftInfoError> { let mut res_list = Vec::new(); - - let (coin_str, chain_str) = chain.to_ticker_chain(); - let my_address = get_eth_address(ctx, &coin_str).await?; - let uri_without_cursor = format!( - "{}{}/nft?chain={}&{}", - URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS - ); + let my_address = get_eth_address(ctx, &chain.to_ticker()).await?; + + let mut uri_without_cursor = url.clone(); + uri_without_cursor.set_path(MORALIS_API_ENDPOINT); + uri_without_cursor + .path_segments_mut() + .map_to_mm(|_| GetNftInfoError::Internal("Invalid URI".to_string()))? + .push(&my_address.wallet_address) + .push("nft"); + uri_without_cursor + .query_pairs_mut() + .append_pair("chain", &chain.to_string()) + .append_pair(MORALIS_FORMAT_QUERY_NAME, MORALIS_FORMAT_QUERY_VALUE); + drop_mutability!(uri_without_cursor); // The cursor returned in the previous response (used for getting the next page). let mut cursor = String::new(); loop { let uri = format!("{}{}", uri_without_cursor, cursor); - let response = send_moralis_request(uri.as_str()).await?; + let response = send_request_to_uri(uri.as_str()).await?; if let Some(nfts_list) = response["result"].as_array() { for nft_json in nfts_list { let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; + let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; let nft = Nft { chain: *chain, token_address: nft_wrapper.token_address, @@ -298,7 +245,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, block_number_minted: *nft_wrapper.block_number_minted, block_number: *nft_wrapper.block_number, contract_type: nft_wrapper.contract_type.map(|v| v.0), - name: nft_wrapper.name, + collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, token_uri: nft_wrapper.token_uri, metadata: nft_wrapper.metadata, @@ -306,6 +253,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain) -> MmResult, last_metadata_sync: nft_wrapper.last_metadata_sync, minter_address: nft_wrapper.minter_address, possible_spam: nft_wrapper.possible_spam, + uri_meta, }; // collect NFTs from the page res_list.push(nft); @@ -329,39 +277,44 @@ async fn get_moralis_nft_transfers( ctx: &MmArc, chain: &Chain, from_block: Option, + url: &Url, ) -> MmResult, GetNftInfoError> { - let api_key = ctx.conf["api_key"] - .as_str() - .ok_or_else(|| MmError::new(GetNftInfoError::ApiKeyError))?; - let mut res_list = Vec::new(); - let (coin_str, chain_str) = match chain { - Chain::Avalanche => ("AVAX", "AVALANCHE"), - Chain::Bsc => ("BNB", "BSC"), - Chain::Eth => ("ETH", "ETH"), - Chain::Fantom => ("FTM", "FANTOM"), - Chain::Polygon => ("MATIC", "POLYGON"), - }; - let my_address = get_eth_address(ctx, coin_str).await?; + let my_address = get_eth_address(ctx, &chain.to_ticker()).await?; + + let mut uri_without_cursor = url.clone(); + uri_without_cursor.set_path(MORALIS_API_ENDPOINT); + uri_without_cursor + .path_segments_mut() + .map_to_mm(|_| GetNftInfoError::Internal("Invalid URI".to_string()))? + .push(&my_address.wallet_address) + .push("nft") + .push("transfers"); let from_block = match from_block { - Some(block) => { - format!("&from_block={}", block) - }, - None => "".into(), + Some(block) => block.to_string(), + None => "0".into(), }; - let uri_without_cursor = format!( - "{}{}/nft/transfers?chain={}&{}&{}{}", - URL_MORALIS, my_address.wallet_address, chain_str, FORMAT_DECIMAL_MORALIS, DIRECTION_BOTH_MORALIS, from_block - ); + uri_without_cursor + .query_pairs_mut() + .append_pair("chain", &chain.to_string()) + .append_pair(MORALIS_FORMAT_QUERY_NAME, MORALIS_FORMAT_QUERY_VALUE) + .append_pair(MORALIS_DIRECTION_QUERY_NAME, MORALIS_DIRECTION_QUERY_VALUE) + .append_pair(MORALIS_FROM_BLOCK_QUERY_NAME, &from_block); + drop_mutability!(uri_without_cursor); // The cursor returned in the previous response (used for getting the next page). let mut cursor = String::new(); loop { let uri = format!("{}{}", uri_without_cursor, cursor); - let response = send_moralis_request(uri.as_str()).await?; + let response = send_request_to_uri(uri.as_str()).await?; if let Some(transfer_list) = response["result"].as_array() { for transfer in transfer_list { let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; + let status = if my_address.wallet_address.to_lowercase() == transfer_wrapper.to_address { + TransferStatus::Receive + } else { + TransferStatus::Send + }; let transfer_history = NftTransferHistory { chain: *chain, block_number: *transfer_wrapper.block_number, @@ -375,8 +328,10 @@ async fn get_moralis_nft_transfers( transaction_type: transfer_wrapper.transaction_type, token_address: transfer_wrapper.token_address, token_id: transfer_wrapper.token_id.0, + collection_name: None, from_address: transfer_wrapper.from_address, to_address: transfer_wrapper.to_address, + status, amount: transfer_wrapper.amount.0, verified: transfer_wrapper.verified, operator: transfer_wrapper.operator, @@ -400,18 +355,91 @@ async fn get_moralis_nft_transfers( Ok(res_list) } +/// **Caution:** ERC-1155 token can have a total supply more than 1, which means there could be several owners +/// of the same token. `get_nft_metadata` returns NFTs info with the most recent owner. +/// **Dont** use this function to get specific info about owner address, amount etc, you will get info not related to my_address. +async fn get_moralis_metadata( + token_address: String, + token_id: BigDecimal, + chain: &Chain, + url: &Url, +) -> MmResult { + let mut uri = url.clone(); + uri.set_path(MORALIS_API_ENDPOINT); + uri.path_segments_mut() + .map_to_mm(|_| GetNftInfoError::Internal("Invalid URI".to_string()))? + .push("nft") + .push(&token_address) + .push(&token_id.to_string()); + uri.query_pairs_mut() + .append_pair("chain", &chain.to_string()) + .append_pair(MORALIS_FORMAT_QUERY_NAME, MORALIS_FORMAT_QUERY_VALUE); + drop_mutability!(uri); + + let response = send_request_to_uri(uri.as_str()).await?; + let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string())?; + let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; + let nft_metadata = Nft { + chain: *chain, + token_address: nft_wrapper.token_address, + token_id: nft_wrapper.token_id.0, + amount: nft_wrapper.amount.0, + owner_of: nft_wrapper.owner_of, + token_hash: nft_wrapper.token_hash, + block_number_minted: *nft_wrapper.block_number_minted, + block_number: *nft_wrapper.block_number, + contract_type: nft_wrapper.contract_type.map(|v| v.0), + collection_name: nft_wrapper.name, + symbol: nft_wrapper.symbol, + token_uri: nft_wrapper.token_uri, + metadata: nft_wrapper.metadata, + last_token_uri_sync: nft_wrapper.last_token_uri_sync, + last_metadata_sync: nft_wrapper.last_metadata_sync, + minter_address: nft_wrapper.minter_address, + possible_spam: nft_wrapper.possible_spam, + uri_meta, + }; + Ok(nft_metadata) +} + /// `withdraw_nft` function generates, signs and returns a transaction that transfers NFT /// from my address to recipient's address. /// This method generates a raw transaction which should then be broadcast using `send_raw_transaction`. pub async fn withdraw_nft(ctx: MmArc, req: WithdrawNftReq) -> WithdrawNftResult { match req.withdraw_type { - WithdrawNftType::WithdrawErc1155(erc1155_withdraw) => withdraw_erc1155(ctx, erc1155_withdraw, req.url).await, + WithdrawNftType::WithdrawErc1155(erc1155_withdraw) => withdraw_erc1155(ctx, erc1155_withdraw).await, WithdrawNftType::WithdrawErc721(erc721_withdraw) => withdraw_erc721(ctx, erc721_withdraw).await, } } +#[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] +enum GetInfoFromUriError { + /// `http::Error` can appear on an HTTP request [`http::Builder::build`] building. + #[from_stringify("http::Error")] + #[display(fmt = "Invalid request: {}", _0)] + InvalidRequest(String), + #[display(fmt = "Transport: {}", _0)] + Transport(String), + #[from_stringify("serde_json::Error")] + #[display(fmt = "Invalid response: {}", _0)] + InvalidResponse(String), + #[display(fmt = "Internal: {}", _0)] + Internal(String), +} + +impl From for GetInfoFromUriError { + fn from(e: SlurpError) -> Self { + let error_str = e.to_string(); + match e { + SlurpError::ErrorDeserializing { .. } => GetInfoFromUriError::InvalidResponse(error_str), + SlurpError::Transport { .. } | SlurpError::Timeout { .. } => GetInfoFromUriError::Transport(error_str), + SlurpError::Internal(_) | SlurpError::InvalidRequest(_) => GetInfoFromUriError::Internal(error_str), + } + } +} + #[cfg(not(target_arch = "wasm32"))] -async fn send_moralis_request(uri: &str) -> MmResult { +async fn send_request_to_uri(uri: &str) -> MmResult { use http::header::HeaderValue; use mm2_net::transport::slurp_req_body; @@ -423,7 +451,7 @@ async fn send_moralis_request(uri: &str) -> MmResult { let (status, _header, body) = slurp_req_body(request).await?; if !status.is_success() { - return Err(MmError::new(GetNftInfoError::Transport(format!( + return Err(MmError::new(GetInfoFromUriError::Transport(format!( "Response !200 from {}: {}, {}", uri, status, body )))); @@ -432,14 +460,14 @@ async fn send_moralis_request(uri: &str) -> MmResult { } #[cfg(target_arch = "wasm32")] -async fn send_moralis_request(uri: &str) -> MmResult { +async fn send_request_to_uri(uri: &str) -> MmResult { use mm2_net::wasm_http::FetchRequest; macro_rules! try_or { ($exp:expr, $errtype:ident) => { match $exp { Ok(x) => x, - Err(e) => return Err(MmError::new(GetNftInfoError::$errtype(ERRL!("{:?}", e)))), + Err(e) => return Err(MmError::new(GetInfoFromUriError::$errtype(ERRL!("{:?}", e)))), } }; } @@ -450,7 +478,7 @@ async fn send_moralis_request(uri: &str) -> MmResult { .await; let (status_code, response_str) = try_or!(result, Transport); if !status_code.is_success() { - return Err(MmError::new(GetNftInfoError::Transport(ERRL!( + return Err(MmError::new(GetInfoFromUriError::Transport(ERRL!( "!200: {}, {}", status_code, response_str @@ -461,6 +489,20 @@ async fn send_moralis_request(uri: &str) -> MmResult { Ok(response) } +async fn try_get_uri_meta(token_uri: &Option) -> MmResult { + match token_uri { + Some(token_uri) => { + if let Ok(response_meta) = send_request_to_uri(token_uri).await { + let uri_meta_res: UriMeta = serde_json::from_str(&response_meta.to_string())?; + Ok(uri_meta_res) + } else { + Ok(UriMeta::default()) + } + }, + None => Ok(UriMeta::default()), + } +} + /// `update_nft_list` function gets nft transfers from NFT HISTORY table, iterates through them /// and updates NFT LIST table info. async fn update_nft_list( @@ -468,6 +510,7 @@ async fn update_nft_list( storage: &T, chain: &Chain, scan_from_block: u32, + url: &Url, ) -> MmResult<(), UpdateNftError> where T: NftListStorageOps + NftTxHistoryStorageOps, @@ -485,13 +528,16 @@ where .await?; }, (false, ContractType::Erc721) => { - let mut nft = get_moralis_metadata(&ctx, tx.token_address, tx.token_id, chain).await?; + let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later // than History by Wallet update nft.owner_of = my_address; nft.block_number = tx.block_number; drop_mutability!(nft); - storage.add_nfts_to_list(chain, [nft], tx.block_number as u32).await?; + storage + .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) + .await?; + update_coll_name_in_txs(storage, chain, vec![nft]).await?; }, (true, ContractType::Erc1155) => { let nft_db = storage @@ -536,7 +582,8 @@ where drop_mutability!(nft_db); storage.update_nft_amount_and_block_number(chain, nft_db).await?; } else { - let moralis_meta = get_moralis_metadata(&ctx, tx.token_address, tx.token_id.clone(), chain).await?; + let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; + let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; let nft = Nft { chain: *chain, token_address: moralis_meta.token_address, @@ -547,7 +594,7 @@ where block_number_minted: moralis_meta.block_number_minted, block_number: tx.block_number, contract_type: moralis_meta.contract_type, - name: moralis_meta.name, + collection_name: moralis_meta.collection_name, symbol: moralis_meta.symbol, token_uri: moralis_meta.token_uri, metadata: moralis_meta.metadata, @@ -555,8 +602,12 @@ where last_metadata_sync: moralis_meta.last_metadata_sync, minter_address: moralis_meta.minter_address, possible_spam: moralis_meta.possible_spam, + uri_meta, }; - storage.add_nfts_to_list(chain, [nft], tx.block_number as u32).await?; + storage + .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) + .await?; + update_coll_name_in_txs(storage, chain, vec![nft]).await?; } }, } diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 2b0aead8ab..4b052e0ff3 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,4 +1,5 @@ use crate::eth::GetEthAddressError; +use crate::nft::GetInfoFromUriError; use crate::nft_storage::{CreateNftStorageError, NftStorageError}; use crate::GetMyAddressError; use common::HttpStatusCode; @@ -35,11 +36,6 @@ pub enum GetNftInfoError { }, #[display(fmt = "DB error {}", _0)] DbError(String), - GetMyAddressError(GetMyAddressError), -} - -impl From for GetNftInfoError { - fn from(e: GetMyAddressError) -> Self { GetNftInfoError::GetMyAddressError(e) } } impl From for GetNftInfoError { @@ -85,6 +81,17 @@ impl From for GetNftInfoError { } } +impl From for GetNftInfoError { + fn from(e: GetInfoFromUriError) -> Self { + match e { + GetInfoFromUriError::InvalidRequest(e) => GetNftInfoError::InvalidRequest(e), + GetInfoFromUriError::Transport(e) => GetNftInfoError::Transport(e), + GetInfoFromUriError::InvalidResponse(e) => GetNftInfoError::InvalidResponse(e), + GetInfoFromUriError::Internal(e) => GetNftInfoError::Internal(e), + } + } +} + impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { @@ -94,8 +101,7 @@ impl HttpStatusCode for GetNftInfoError { | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) | GetNftInfoError::TokenNotFoundInWallet { .. } - | GetNftInfoError::DbError(_) - | GetNftInfoError::GetMyAddressError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 7adae40783..7709f8c152 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -1,9 +1,10 @@ use crate::{TransactionType, TxFeeDetails, WithdrawFee}; -use ethereum_types::Address; use common::ten; +use ethereum_types::Address; use mm2_number::BigDecimal; use rpc::v1::types::Bytes as BytesJson; use serde::Deserialize; +use serde_json::Value as Json; use std::fmt; use std::num::NonZeroUsize; use std::str::FromStr; @@ -12,7 +13,6 @@ use url::Url; #[derive(Debug, Deserialize)] pub struct NftListReq { pub(crate) chains: Vec, - pub(crate) url: Url, #[serde(default)] pub(crate) max: bool, #[serde(default = "ten")] @@ -92,7 +92,7 @@ pub(crate) enum ParseContractTypeError { UnsupportedContractType, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "UPPERCASE")] pub(crate) enum ContractType { Erc1155, @@ -121,7 +121,17 @@ impl fmt::Display for ContractType { } } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct UriMeta { + image: Option, + #[serde(rename(deserialize = "name"))] + token_name: Option, + description: Option, + attributes: Option, + animation_url: Option, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct Nft { pub(crate) chain: Chain, pub(crate) token_address: String, @@ -132,7 +142,7 @@ pub struct Nft { pub(crate) block_number_minted: u64, pub(crate) block_number: u64, pub(crate) contract_type: Option, - pub(crate) name: Option, + pub(crate) collection_name: Option, pub(crate) symbol: Option, pub(crate) token_uri: Option, pub(crate) metadata: Option, @@ -140,6 +150,7 @@ pub struct Nft { pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, pub(crate) possible_spam: Option, + pub(crate) uri_meta: UriMeta, } /// This structure is for deserializing NFT json to struct. @@ -217,7 +228,6 @@ pub struct WithdrawErc721 { #[derive(Clone, Deserialize)] pub struct WithdrawNftReq { - pub(crate) url: Url, pub(crate) withdraw_type: WithdrawNftType, } @@ -259,7 +269,6 @@ pub struct TransactionNftDetails { #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, - pub(crate) url: Url, pub(crate) filters: Option, #[serde(default)] pub(crate) max: bool, @@ -268,6 +277,39 @@ pub struct NftTransfersReq { pub(crate) page_number: Option, } +#[derive(Debug, Display)] +pub(crate) enum ParseTransferStatusError { + UnsupportedTransferStatus, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(crate) enum TransferStatus { + Receive, + Send, +} + +impl FromStr for TransferStatus { + type Err = ParseTransferStatusError; + + #[inline] + fn from_str(s: &str) -> Result { + match s { + "Receive" => Ok(TransferStatus::Receive), + "Send" => Ok(TransferStatus::Send), + _ => Err(ParseTransferStatusError::UnsupportedTransferStatus), + } + } +} + +impl fmt::Display for TransferStatus { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + TransferStatus::Receive => write!(f, "Receive"), + TransferStatus::Send => write!(f, "Send"), + } + } +} + #[derive(Debug, Deserialize, Serialize)] pub struct NftTransferHistory { pub(crate) chain: Chain, @@ -283,8 +325,10 @@ pub struct NftTransferHistory { pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: BigDecimal, + pub(crate) collection_name: Option, pub(crate) from_address: String, pub(crate) to_address: String, + pub(crate) status: TransferStatus, pub(crate) amount: BigDecimal, pub(crate) verified: u64, pub(crate) operator: Option, @@ -333,4 +377,5 @@ pub struct NftTxHistoryFilters { #[derive(Debug, Deserialize)] pub struct UpdateNftReq { pub(crate) chains: Vec, + pub(crate) url: Url, } diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 6c6640cb99..4ab868bad0 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -5,14 +5,14 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { - use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper}; + use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::send_moralis_request; + use crate::nft::send_request_to_uri; use common::block_on; #[test] fn test_moralis_nft_list() { - let response = block_on(send_moralis_request(NFT_LIST_URL_TEST)).unwrap(); + let response = block_on(send_request_to_uri(NFT_LIST_URL_TEST)).unwrap(); let nfts_list = response["result"].as_array().unwrap(); assert_eq!(2, nfts_list.len()); for nft_json in nfts_list { @@ -23,7 +23,7 @@ mod native_tests { #[test] fn test_moralis_nft_transfer_history() { - let response = block_on(send_moralis_request(NFT_HISTORY_URL_TEST)).unwrap(); + let response = block_on(send_request_to_uri(NFT_HISTORY_URL_TEST)).unwrap(); let mut transfer_list = response["result"].as_array().unwrap().clone(); assert_eq!(4, transfer_list.len()); let last_tx = transfer_list.remove(0); @@ -33,9 +33,12 @@ mod native_tests { #[test] fn test_moralis_nft_metadata() { - let response = block_on(send_moralis_request(NFT_METADATA_URL_TEST)).unwrap(); + let response = block_on(send_request_to_uri(NFT_METADATA_URL_TEST)).unwrap(); let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); - assert_eq!(41237364, *nft_wrapper.block_number_minted) + assert_eq!(41237364, *nft_wrapper.block_number_minted); + let token_uri = nft_wrapper.token_uri.unwrap(); + let uri_response = block_on(send_request_to_uri(token_uri.as_str())).unwrap(); + serde_json::from_str::(&uri_response.to_string()).unwrap(); } } @@ -43,14 +46,14 @@ mod native_tests { mod wasm_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::send_moralis_request; + use crate::nft::send_request_to_uri; use wasm_bindgen_test::*; wasm_bindgen_test_configure!(run_in_browser); #[wasm_bindgen_test] async fn test_moralis_nft_list() { - let response = send_moralis_request(NFT_LIST_URL_TEST).await.unwrap(); + let response = send_request_to_uri(NFT_LIST_URL_TEST).await.unwrap(); let nfts_list = response["result"].as_array().unwrap(); assert_eq!(2, nfts_list.len()); for nft_json in nfts_list { @@ -61,7 +64,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_moralis_nft_transfer_history() { - let response = send_moralis_request(NFT_HISTORY_URL_TEST).await.unwrap(); + let response = send_request_to_uri(NFT_HISTORY_URL_TEST).await.unwrap(); let mut transfer_list = response["result"].as_array().unwrap().clone(); assert_eq!(4, transfer_list.len()); let last_tx = transfer_list.remove(0); @@ -71,8 +74,8 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_moralis_nft_metadata() { - let response = send_moralis_request(NFT_METADATA_URL_TEST).await.unwrap(); + let response = send_request_to_uri(NFT_METADATA_URL_TEST).await.unwrap(); let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); - assert_eq!(41237364, *nft_wrapper.block_number_minted) + assert_eq!(41237364, *nft_wrapper.block_number_minted); } } diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 7abe127bd2..5af28c3f86 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -91,7 +91,7 @@ pub trait NftTxHistoryStorageOps { async fn get_tx_history( &self, - chain_addr: Vec<(Chain, String)>, + chains: Vec, max: bool, limit: usize, page_number: Option, @@ -112,6 +112,29 @@ pub trait NftTxHistoryStorageOps { chain: &Chain, from_block: u32, ) -> MmResult, Self::Error>; + + async fn get_txs_by_token_addr_id( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult, Self::Error>; + + async fn get_tx_by_tx_hash( + &self, + chain: &Chain, + transaction_hash: String, + ) -> MmResult, Self::Error>; + + async fn update_tx_details_json_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error>; + + async fn update_txs_coll_name_by_token_addr_id( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + collection_name: Option, + ) -> MmResult<(), Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 4fb327d452..7ff2b86546 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -55,8 +55,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { contract_type TEXT NOT NULL, token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, - from_address TEXT NOT NULL, - to_address TEXT NOT NULL, + status TEXT NOT NULL, amount VARCHAR(256) NOT NULL, details_json TEXT );", @@ -116,15 +115,15 @@ fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmRes fn get_nft_tx_builder_preimage( conn: &Connection, - chain_addr: Vec<(Chain, String)>, + chains: Vec, filters: Option, ) -> MmResult { - let union_sql_strings: MmResult, SqlError> = chain_addr + let union_sql_strings: MmResult, SqlError> = chains .into_iter() - .map(|(chain, addr)| { + .map(|chain| { let table_name = nft_tx_history_table_name(&chain); validate_table_name(&table_name)?; - let sql_builder = nft_history_table_builder_preimage(conn, table_name.as_str(), addr, filters.clone())?; + let sql_builder = nft_history_table_builder_preimage(conn, table_name.as_str(), filters.clone())?; let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); Ok(sql_string) }) @@ -141,21 +140,14 @@ fn get_nft_tx_builder_preimage( fn nft_history_table_builder_preimage<'a>( conn: &'a Connection, table_name: &'a str, - owner_add: String, filters: Option, ) -> Result, SqlError> { let mut sql_builder = SqlQuery::select_from(conn, table_name)?; if let Some(filters) = filters { - let owner_add = format!("'{}'", owner_add); - if filters.send && filters.receive { - sql_builder - .sql_builder() - .and_where_eq("from_address", owner_add.clone()) - .or_where_eq("to_address", owner_add); - } else if filters.send { - sql_builder.sql_builder().and_where_eq("from_address", owner_add); - } else if filters.receive { - sql_builder.sql_builder().and_where_eq("to_address", owner_add); + if filters.send && !filters.receive { + sql_builder.sql_builder().and_where_eq("status", "Send"); + } else if filters.receive && !filters.send { + sql_builder.sql_builder().and_where_eq("status", "Receive"); } if filters.from_date.is_some() && filters.to_date.is_some() { @@ -228,9 +220,9 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT OR IGNORE INTO {} ( transaction_hash, chain, block_number, block_timestamp, contract_type, - token_address, token_id, from_address, to_address, amount, details_json + token_address, token_id, status, amount, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10 );", table_name ); @@ -243,7 +235,7 @@ fn insert_chain_into_scanned_blocks_sql() -> MmResult { Ok(format!("INSERT OR IGNORE INTO {} (chain) VALUES (?1);", table_name)) } -fn update_details_json_sql(chain: &Chain, table_name_creator: F) -> MmResult +fn update_details_json_by_token_add_id_sql(chain: &Chain, table_name_creator: F) -> MmResult where F: FnOnce(&Chain) -> String, { @@ -257,6 +249,17 @@ where Ok(sql) } +fn update_details_json_by_tx_hash_sql(chain: &Chain) -> MmResult { + let table_name = nft_tx_history_table_name(chain); + + validate_table_name(&table_name)?; + let sql = format!( + "UPDATE {} SET details_json = ?1 WHERE transaction_hash = ?2;", + table_name + ); + Ok(sql) +} + fn update_nft_amount_sql(chain: &Chain, table_name_creator: F) -> MmResult where F: FnOnce(&Chain) -> String, @@ -366,6 +369,31 @@ fn get_txs_from_block_builder<'a>( Ok(sql_builder) } +fn get_txs_by_token_addr_id_builder<'a>( + conn: &'a Connection, + chain: &'a Chain, + token_address: String, + token_id: String, +) -> MmResult, SqlError> { + let table_name = nft_tx_history_table_name(chain); + validate_table_name(table_name.as_str())?; + let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; + sql_builder + .sql_builder() + .and_where_eq("token_address", format!("'{}'", token_address)) + .and_where_eq("token_id", format!("'{}'", token_id)) + .field("details_json"); + drop_mutability!(sql_builder); + Ok(sql_builder) +} + +fn get_tx_by_tx_hash_sql(chain: &Chain) -> MmResult { + let table_name = nft_tx_history_table_name(chain); + validate_table_name(&table_name)?; + let sql = format!("SELECT details_json FROM {} WHERE transaction_hash=?1", table_name); + Ok(sql) +} + #[async_trait] impl NftListStorageOps for SqliteNftStorage { type Error = SqlError; @@ -534,13 +562,13 @@ impl NftListStorageOps for SqliteNftStorage { } async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { - let sql = update_details_json_sql(chain, nft_list_table_name)?; + let sql = update_details_json_by_token_add_id_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - let params = [Some(nft_json), Some(nft.token_address), Some(nft.token_id.to_string())]; + let params = [nft_json, nft.token_address, nft.token_id.to_string()]; sql_transaction.execute(&sql, ¶ms)?; sql_transaction.commit()?; Ok(()) @@ -648,7 +676,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async fn get_tx_history( &self, - chain_addr: Vec<(Chain, String)>, + chains: Vec, max: bool, limit: usize, page_number: Option, @@ -657,7 +685,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut sql_builder = get_nft_tx_builder_preimage(&conn, chain_addr, filters)?; + let mut sql_builder = get_nft_tx_builder_preimage(&conn, chains, filters)?; let mut total_count_builder = sql_builder.clone(); total_count_builder.count_all()?; let total: isize = total_count_builder @@ -705,8 +733,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.contract_type.to_string()), Some(tx.token_address), Some(tx.token_id.to_string()), - Some(tx.from_address), - Some(tx.to_address), + Some(tx.status.to_string()), Some(tx.amount.to_string()), Some(tx_json), ]; @@ -743,4 +770,68 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { }) .await } + + async fn get_txs_by_token_addr_id( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + ) -> MmResult, Self::Error> { + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let sql_builder = get_txs_by_token_addr_id_builder(&conn, &chain, token_address, token_id.to_string())?; + let txs = sql_builder.query(tx_history_from_row)?; + Ok(txs) + }) + .await + } + + async fn get_tx_by_tx_hash( + &self, + chain: &Chain, + transaction_hash: String, + ) -> MmResult, Self::Error> { + let sql = get_tx_by_tx_hash_sql(chain)?; + let selfi = self.clone(); + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + query_single_row(&conn, &sql, [transaction_hash], tx_history_from_row).map_to_mm(SqlError::from) + }) + .await + } + + async fn update_tx_details_json_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error> { + let sql = update_details_json_by_tx_hash_sql(chain)?; + let tx_json = json::to_string(&tx).expect("serialization should not fail"); + let params = [tx_json, tx.transaction_hash]; + let selfi = self.clone(); + async_blocking(move || { + let mut conn = selfi.0.lock().unwrap(); + let sql_transaction = conn.transaction()?; + + sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.commit()?; + Ok(()) + }) + .await + } + + async fn update_txs_coll_name_by_token_addr_id( + &self, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + collection_name: Option, + ) -> MmResult<(), Self::Error> { + let selfi = self.clone(); + let txs = selfi.get_txs_by_token_addr_id(chain, token_address, token_id).await?; + for mut tx in txs.into_iter() { + tx.collection_name = collection_name.clone(); + drop_mutability!(tx); + selfi.update_tx_details_json_by_hash(chain, tx).await?; + } + Ok(()) + } } diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index b7b331cd9d..d0d0cd9567 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,6 +1,5 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageError, NftListStorageOps, NftTxHistoryStorageError, - NftTxHistoryStorageOps}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; use crate::CoinsContext; use async_trait::async_trait; use derive_more::Display; @@ -11,6 +10,8 @@ use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, DbTransactionError, use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; +use mm2_number::BigDecimal; +use std::num::NonZeroUsize; const DB_NAME: &str = "nft_cache"; const DB_VERSION: u32 = 1; @@ -18,8 +19,7 @@ const DB_VERSION: u32 = 1; pub type WasmNftCacheResult = MmResult; pub type NftCacheIDBLocked<'a> = DbLocked<'a, NftCacheIDB>; -impl NftListStorageError for WasmNftCacheError {} -impl NftTxHistoryStorageError for WasmNftCacheError {} +impl NftStorageError for WasmNftCacheError {} #[derive(Debug, Display)] pub enum WasmNftCacheError { @@ -113,11 +113,19 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } + async fn is_initialized(&self, _chain: &Chain) -> MmResult { todo!() } - async fn get_nft_list(&self, _chain: &Chain) -> MmResult { todo!() } + async fn get_nft_list( + &self, + _chains: Vec, + _max: bool, + _limit: usize, + _page_number: Option, + ) -> MmResult { + todo!() + } - async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I, _last_scanned_block: u32) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -125,7 +133,36 @@ impl NftListStorageOps for IndexedDbNftStorage { todo!() } - async fn remove_nft_from_list(&self, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } + async fn remove_nft_from_list( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + _scanned_block: u64, + ) -> MmResult { + todo!() + } + + async fn get_nft_amount( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + ) -> MmResult, Self::Error> { + todo!() + } + + async fn refresh_nft_metadata(&self, _chain: &Chain, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } + + async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + + async fn update_nft_amount(&self, _chain: &Chain, _nft: Nft, _scanned_block: u64) -> MmResult<(), Self::Error> { + todo!() + } + + async fn update_nft_amount_and_block_number(&self, _chain: &Chain, _nft: Nft) -> MmResult<(), Self::Error> { + todo!() + } } #[async_trait] @@ -134,7 +171,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { todo!() } - async fn is_initialized_for(&self, _chain: &Chain) -> MmResult { todo!() } + async fn is_initialized(&self, _chain: &Chain) -> MmResult { todo!() } async fn get_tx_history(&self, _chain: &Chain) -> MmResult { todo!() } From cb29933d056a1ab46197dcfa0e98bfd1ea654997 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 22 May 2023 11:20:59 +0700 Subject: [PATCH 046/112] find wallet_amount in withdraw_erc1155 --- mm2src/coins/eth.rs | 12 ++++++++---- mm2src/coins/lp_coins.rs | 22 +++++++++++++++++++++- mm2src/coins/nft.rs | 29 +++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/eth.rs b/mm2src/coins/eth.rs index 3d46f00868..59c3c73a1f 100644 --- a/mm2src/coins/eth.rs +++ b/mm2src/coins/eth.rs @@ -105,7 +105,7 @@ pub use rlp; mod web3_transport; #[path = "eth/v2_activation.rs"] pub mod v2_activation; -use crate::nft::WithdrawNftResult; +use crate::nft::{find_wallet_nft_amount, WithdrawNftResult}; use v2_activation::{build_address_and_priv_key_policy, EthActivationV2Error}; mod nonce; @@ -879,9 +879,13 @@ pub async fn withdraw_erc1155(ctx: MmArc, withdraw_type: WithdrawErc1155) -> Wit get_valid_nft_add_to_withdraw(coin, &withdraw_type.to, &withdraw_type.token_address)?; let my_address = eth_coin.my_address()?; - // todo check amount in nft cache, instead of sending new moralis req - let wallet_amount = BigDecimal::default(); - + let wallet_amount = find_wallet_nft_amount( + &ctx, + &withdraw_type.chain, + withdraw_type.token_address.to_lowercase(), + withdraw_type.token_id.clone(), + ) + .await?; let amount_dec = if withdraw_type.max { wallet_amount.clone() } else { diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 9fc38d400a..70b9ceb632 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -292,6 +292,7 @@ pub mod nft_storage; #[cfg(not(target_arch = "wasm32"))] pub mod z_coin; +use crate::nft_storage::{CreateNftStorageError, NftStorageError}; #[cfg(not(target_arch = "wasm32"))] use z_coin::ZCoin; pub type TransactionFut = Box + Send>; @@ -1929,12 +1930,29 @@ pub enum WithdrawError { available: BigDecimal, required: BigDecimal, }, + #[display(fmt = "DB error {}", _0)] + DbError(String), } impl From for WithdrawError { fn from(e: GetNftInfoError) -> Self { WithdrawError::GetNftInfoError(e) } } +impl From for WithdrawError { + fn from(err: T) -> Self { + let msg = format!("{:?}", err); + WithdrawError::DbError(msg) + } +} + +impl From for WithdrawError { + fn from(e: CreateNftStorageError) -> Self { + match e { + CreateNftStorageError::Internal(err) => WithdrawError::InternalError(err), + } + } +} + impl HttpStatusCode for WithdrawError { fn status_code(&self) -> StatusCode { match self { @@ -1961,7 +1979,9 @@ impl HttpStatusCode for WithdrawError { WithdrawError::HwError(_) => StatusCode::GONE, #[cfg(target_arch = "wasm32")] WithdrawError::BroadcastExpected(_) => StatusCode::BAD_REQUEST, - WithdrawError::Transport(_) | WithdrawError::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, + WithdrawError::Transport(_) | WithdrawError::InternalError(_) | WithdrawError::DbError(_) => { + StatusCode::INTERNAL_SERVER_ERROR + }, } } } diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 81bd3190b2..2cb3d6a5ba 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -614,3 +614,32 @@ where } Ok(()) } + +/// `find_wallet_nft_amount` function returns NFT amount of cached NFT. +/// In db token_address is kept in lowercase, because Moralis returns all addresses in lowercase. +pub(crate) async fn find_wallet_nft_amount( + ctx: &MmArc, + chain: &Chain, + token_address: String, + token_id: BigDecimal, +) -> MmResult { + let storage = NftStorageBuilder::new(ctx).build()?; + if !NftListStorageOps::is_initialized(&storage, chain).await? { + NftListStorageOps::init(&storage, chain).await?; + } + + let nft_meta = storage + .get_nft(chain, token_address.to_lowercase(), token_id.clone()) + .await?; + + let wallet_amount = match nft_meta { + Some(nft) => nft.amount, + None => { + return MmError::err(GetNftInfoError::TokenNotFoundInWallet { + token_address, + token_id: token_id.to_string(), + }) + }, + }; + Ok(wallet_amount) +} From 7941fe3b0e4bd0a7a730b15fd2d79655bdbb9c93 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 22 May 2023 11:58:16 +0700 Subject: [PATCH 047/112] todos in wasm target --- mm2src/coins/nft/nft_structs.rs | 1 + mm2src/coins/nft_storage/wasm_storage.rs | 76 ++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 7709f8c152..4e718978aa 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -364,6 +364,7 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } +#[allow(dead_code)] #[derive(Clone, Debug, Deserialize)] pub struct NftTxHistoryFilters { #[serde(default)] diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index d0d0cd9567..c7754c60c4 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,5 +1,6 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryFilters, + NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; use async_trait::async_trait; use derive_more::Display; @@ -79,12 +80,13 @@ impl DbInstance for NftCacheIDB { fn db_name() -> &'static str { DB_NAME } async fn init(db_id: DbIdentifier) -> InitDbResult { - // todo add tables for each chain? + // todo add tables for each chain let inner = IndexedDbBuilder::new(db_id).with_version(DB_VERSION).build().await?; Ok(NftCacheIDB { inner }) } } +#[allow(dead_code)] impl NftCacheIDB { fn get_inner(&self) -> &IndexedDb { &self.inner } } @@ -102,6 +104,7 @@ impl IndexedDbNftStorage { }) } + #[allow(dead_code)] async fn lock_db(&self) -> WasmNftCacheResult> { self.db.get_or_initialize().await.mm_err(WasmNftCacheError::from) } @@ -133,6 +136,15 @@ impl NftListStorageOps for IndexedDbNftStorage { todo!() } + async fn get_nft( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + ) -> MmResult, Self::Error> { + todo!() + } + async fn remove_nft_from_list( &self, _chain: &Chain, @@ -156,6 +168,8 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_last_scanned_block(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn update_nft_amount(&self, _chain: &Chain, _nft: Nft, _scanned_block: u64) -> MmResult<(), Self::Error> { todo!() } @@ -173,13 +187,67 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn is_initialized(&self, _chain: &Chain) -> MmResult { todo!() } - async fn get_tx_history(&self, _chain: &Chain) -> MmResult { todo!() } + async fn get_tx_history( + &self, + _chains: Vec, + _max: bool, + _limit: usize, + _page_number: Option, + _filters: Option, + ) -> MmResult { + todo!() + } - async fn add_txs_to_history(&self, _chain: &Chain, _nfts: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, _chain: &Chain, _txs: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { todo!() } + + async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + + async fn get_txs_from_block( + &self, + _chain: &Chain, + _from_block: u32, + ) -> MmResult, Self::Error> { + todo!() + } + + async fn get_txs_by_token_addr_id( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + ) -> MmResult, Self::Error> { + todo!() + } + + async fn get_tx_by_tx_hash( + &self, + _chain: &Chain, + _transaction_hash: String, + ) -> MmResult, Self::Error> { + todo!() + } + + async fn update_tx_details_json_by_hash( + &self, + _chain: &Chain, + _tx: NftTransferHistory, + ) -> MmResult<(), Self::Error> { + todo!() + } + + async fn update_txs_coll_name_by_token_addr_id( + &self, + _chain: &Chain, + _token_address: String, + _token_id: BigDecimal, + _collection_name: Option, + ) -> MmResult<(), Self::Error> { + todo!() + } } From 3323643bbcd503754baa4f88f6c42c25b855c9c6 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 22 May 2023 17:02:27 +0700 Subject: [PATCH 048/112] update empty meta in txs --- mm2src/coins/nft.rs | 244 +++++++++++++++++------ mm2src/coins/nft/nft_structs.rs | 20 +- mm2src/coins/nft_storage/mod.rs | 11 +- mm2src/coins/nft_storage/sql_storage.rs | 80 ++++++-- mm2src/coins/nft_storage/wasm_storage.rs | 4 +- 5 files changed, 270 insertions(+), 89 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 2cb3d6a5ba..45cabbbee7 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -13,7 +13,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, N TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::{TransferStatus, UriMeta, WithdrawNftType}; +use crate::nft::nft_structs::{TransferStatus, UriMeta}; use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::APPLICATION_JSON; use enum_from::EnumFromStringify; @@ -22,6 +22,7 @@ use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_net::transport::SlurpError; use mm2_number::BigDecimal; use serde_json::Value as Json; +use std::cmp::Ordering; const MORALIS_API_ENDPOINT: &str = "api/v2"; /// query parameters for moralis request: The format of the token ID @@ -108,7 +109,9 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft storage .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) .await?; - update_coll_name_in_txs(&storage, chain, nft_list).await?; + // this will update only txs related to current nfts in wallet. + update_meta_in_txs(&storage, chain, nft_list).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; } else { let last_scanned_block = storage.get_last_scanned_block(chain).await?; let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; @@ -119,6 +122,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft (Some(scanned_block), Some(nft_block)) => { if scanned_block >= nft_block { update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; } else { return MmError::err(UpdateNftError::InvalidBlockOrder { last_scanned_block: scanned_block.to_string(), @@ -138,11 +142,13 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft // we can try to get all info from moralis. (Some(_), None) => { let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; - update_coll_name_in_txs(&storage, chain, nfts).await?; + update_meta_in_txs(&storage, chain, nfts).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; }, (None, None) => { let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; - update_coll_name_in_txs(&storage, chain, nfts).await?; + update_meta_in_txs(&storage, chain, nfts).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; }, } } @@ -150,37 +156,6 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft Ok(()) } -async fn cache_nfts_from_moralis( - ctx: &MmArc, - storage: &T, - chain: &Chain, - url: &Url, -) -> MmResult, UpdateNftError> -where - T: NftListStorageOps + NftTxHistoryStorageOps, -{ - let nft_list = get_moralis_nft_list(ctx, chain, url).await?; - let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(storage, chain) - .await? - .unwrap_or(0); - storage - .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) - .await?; - Ok(nft_list) -} - -async fn update_coll_name_in_txs(storage: &T, chain: &Chain, nfts: Vec) -> MmResult<(), UpdateNftError> -where - T: NftListStorageOps + NftTxHistoryStorageOps, -{ - for nft in nfts.into_iter() { - storage - .update_txs_coll_name_by_token_addr_id(chain, nft.token_address, nft.token_id, nft.collection_name) - .await?; - } - Ok(()) -} - pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<(), UpdateNftError> { let moralis_meta = get_moralis_metadata( format!("{:#02x}", req.token_address), @@ -196,16 +171,30 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<( chain: req.chain, url: req.url, }; - let mut nft = get_nft_metadata(ctx, req).await?; - nft.collection_name = moralis_meta.collection_name; - nft.symbol = moralis_meta.symbol; - nft.token_uri = moralis_meta.token_uri; - nft.metadata = moralis_meta.metadata; - nft.last_token_uri_sync = moralis_meta.last_token_uri_sync; - nft.last_metadata_sync = moralis_meta.last_metadata_sync; - nft.possible_spam = moralis_meta.possible_spam; - drop_mutability!(nft); - storage.refresh_nft_metadata(&moralis_meta.chain, nft).await?; + let mut nft_db = get_nft_metadata(ctx, req).await?; + let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; + nft_db.collection_name = moralis_meta.collection_name; + nft_db.symbol = moralis_meta.symbol; + nft_db.token_uri = moralis_meta.token_uri; + nft_db.metadata = moralis_meta.metadata; + nft_db.last_token_uri_sync = moralis_meta.last_token_uri_sync; + nft_db.last_metadata_sync = moralis_meta.last_metadata_sync; + nft_db.possible_spam = moralis_meta.possible_spam; + nft_db.uri_meta = uri_meta; + drop_mutability!(nft_db); + storage + .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) + .await?; + storage + .update_txs_meta_by_token_addr_id( + &nft_db.chain, + nft_db.token_address, + nft_db.token_id, + nft_db.collection_name, + nft_db.uri_meta.image, + nft_db.uri_meta.token_name, + ) + .await?; Ok(()) } @@ -329,6 +318,8 @@ async fn get_moralis_nft_transfers( token_address: transfer_wrapper.token_address, token_id: transfer_wrapper.token_id.0, collection_name: None, + image: None, + token_name: None, from_address: transfer_wrapper.from_address, to_address: transfer_wrapper.to_address, status, @@ -406,9 +397,9 @@ async fn get_moralis_metadata( /// from my address to recipient's address. /// This method generates a raw transaction which should then be broadcast using `send_raw_transaction`. pub async fn withdraw_nft(ctx: MmArc, req: WithdrawNftReq) -> WithdrawNftResult { - match req.withdraw_type { - WithdrawNftType::WithdrawErc1155(erc1155_withdraw) => withdraw_erc1155(ctx, erc1155_withdraw).await, - WithdrawNftType::WithdrawErc721(erc721_withdraw) => withdraw_erc721(ctx, erc721_withdraw).await, + match req { + WithdrawNftReq::WithdrawErc1155(erc1155_withdraw) => withdraw_erc1155(ctx, erc1155_withdraw).await, + WithdrawNftReq::WithdrawErc721(erc721_withdraw) => withdraw_erc721(ctx, erc721_withdraw).await, } } @@ -521,13 +512,30 @@ where coin: chain.to_ticker(), }; let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); - match (tx.from_address == my_address, tx.contract_type) { - (true, ContractType::Erc721) => { + match (tx.status, tx.contract_type) { + (TransferStatus::Send, ContractType::Erc721) => { + if let Some(nft) = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await? + { + storage + .update_txs_meta_by_token_addr_id( + chain, + nft.token_address, + nft.token_id, + nft.collection_name, + nft.uri_meta.image, + nft.uri_meta.token_name, + ) + .await?; + } else { + continue; + }; storage .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) .await?; }, - (false, ContractType::Erc721) => { + (TransferStatus::Receive, ContractType::Erc721) => { let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later // than History by Wallet update @@ -537,26 +545,52 @@ where storage .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) .await?; - update_coll_name_in_txs(storage, chain, vec![nft]).await?; + storage + .update_txs_meta_by_token_addr_id( + chain, + nft.token_address, + nft.token_id, + nft.collection_name, + nft.uri_meta.image, + nft.uri_meta.token_name, + ) + .await?; }, - (true, ContractType::Erc1155) => { + (TransferStatus::Send, ContractType::Erc1155) => { let nft_db = storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await?; - // change amount or delete nft from the list + // If nft exists then check the amount if let Some(mut nft_db) = nft_db { match nft_db.amount.cmp(&tx.amount) { - std::cmp::Ordering::Equal => { + Ordering::Equal => { + if let Some(nft) = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await? + { + storage + .update_txs_meta_by_token_addr_id( + chain, + nft.token_address, + nft.token_id, + nft.collection_name, + nft.uri_meta.image, + nft.uri_meta.token_name, + ) + .await?; + } else { + continue; + }; storage .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) .await?; }, - std::cmp::Ordering::Greater => { + Ordering::Greater => { nft_db.amount -= tx.amount; drop_mutability!(nft_db); storage.update_nft_amount(chain, nft_db, tx.block_number).await?; }, - std::cmp::Ordering::Less => { + Ordering::Less => { return MmError::err(UpdateNftError::InsufficientAmountInCache { amount_list: nft_db.amount.to_string(), amount_history: tx.amount.to_string(), @@ -571,16 +605,32 @@ where }); } }, - (false, ContractType::Erc1155) => { + (TransferStatus::Receive, ContractType::Erc1155) => { let nft_db = storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await?; - // change amount and block number or add nft to the list + // If token isn't in NFT LIST table then add nft to the table. if let Some(mut nft_db) = nft_db { - nft_db.amount += tx.amount; + // if owner address == from address, then owner sent tokens to themself, + // which means that the amount will not change. + if my_address != tx.from_address { + nft_db.amount += tx.amount; + } nft_db.block_number = tx.block_number; drop_mutability!(nft_db); - storage.update_nft_amount_and_block_number(chain, nft_db).await?; + storage + .update_nft_amount_and_block_number(chain, nft_db.clone()) + .await?; + storage + .update_txs_meta_by_token_addr_id( + chain, + nft_db.token_address, + nft_db.token_id, + nft_db.collection_name, + nft_db.uri_meta.image, + nft_db.uri_meta.token_name, + ) + .await?; } else { let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; @@ -607,7 +657,16 @@ where storage .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) .await?; - update_coll_name_in_txs(storage, chain, vec![nft]).await?; + storage + .update_txs_meta_by_token_addr_id( + chain, + nft.token_address, + nft.token_id, + nft.collection_name, + nft.uri_meta.image, + nft.uri_meta.token_name, + ) + .await?; } }, } @@ -616,7 +675,7 @@ where } /// `find_wallet_nft_amount` function returns NFT amount of cached NFT. -/// In db token_address is kept in lowercase, because Moralis returns all addresses in lowercase. +/// Note: in db **token_address** is kept in **lowercase**, because Moralis returns all addresses in lowercase. pub(crate) async fn find_wallet_nft_amount( ctx: &MmArc, chain: &Chain, @@ -643,3 +702,62 @@ pub(crate) async fn find_wallet_nft_amount( }; Ok(wallet_amount) } + +async fn cache_nfts_from_moralis( + ctx: &MmArc, + storage: &T, + chain: &Chain, + url: &Url, +) -> MmResult, UpdateNftError> +where + T: NftListStorageOps + NftTxHistoryStorageOps, +{ + let nft_list = get_moralis_nft_list(ctx, chain, url).await?; + let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(storage, chain) + .await? + .unwrap_or(0); + storage + .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) + .await?; + Ok(nft_list) +} + +async fn update_meta_in_txs(storage: &T, chain: &Chain, nfts: Vec) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTxHistoryStorageOps, +{ + for nft in nfts.into_iter() { + storage + .update_txs_meta_by_token_addr_id( + chain, + nft.token_address, + nft.token_id, + nft.collection_name, + nft.uri_meta.image, + nft.uri_meta.token_name, + ) + .await?; + } + Ok(()) +} + +async fn update_txs_with_empty_meta(storage: &T, chain: &Chain, url: &Url) -> MmResult<(), UpdateNftError> +where + T: NftListStorageOps + NftTxHistoryStorageOps, +{ + let nft_token_addr_id = storage.get_txs_with_empty_meta(chain).await?; + for addr_id_pair in nft_token_addr_id.into_iter() { + let nft_meta = get_moralis_metadata(addr_id_pair.token_address, addr_id_pair.token_id, chain, url).await?; + storage + .update_txs_meta_by_token_addr_id( + chain, + nft_meta.token_address, + nft_meta.token_id, + nft_meta.collection_name, + nft_meta.uri_meta.image, + nft_meta.uri_meta.token_name, + ) + .await?; + } + Ok(()) +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 4e718978aa..3708aec138 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -123,9 +123,9 @@ impl fmt::Display for ContractType { #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub(crate) struct UriMeta { - image: Option, + pub(crate) image: Option, #[serde(rename(deserialize = "name"))] - token_name: Option, + pub(crate) token_name: Option, description: Option, attributes: Option, animation_url: Option, @@ -226,15 +226,10 @@ pub struct WithdrawErc721 { pub(crate) fee: Option, } -#[derive(Clone, Deserialize)] -pub struct WithdrawNftReq { - pub(crate) withdraw_type: WithdrawNftType, -} - #[derive(Clone, Deserialize)] #[serde(tag = "type", content = "withdraw_data")] #[serde(rename_all = "snake_case")] -pub enum WithdrawNftType { +pub enum WithdrawNftReq { WithdrawErc1155(WithdrawErc1155), WithdrawErc721(WithdrawErc721), } @@ -326,6 +321,8 @@ pub struct NftTransferHistory { pub(crate) token_address: String, pub(crate) token_id: BigDecimal, pub(crate) collection_name: Option, + pub(crate) image: Option, + pub(crate) token_name: Option, pub(crate) from_address: String, pub(crate) to_address: String, pub(crate) status: TransferStatus, @@ -364,7 +361,6 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } -#[allow(dead_code)] #[derive(Clone, Debug, Deserialize)] pub struct NftTxHistoryFilters { #[serde(default)] @@ -380,3 +376,9 @@ pub struct UpdateNftReq { pub(crate) chains: Vec, pub(crate) url: Url, } + +#[derive(Debug, Deserialize)] +pub struct NftTokenAddrId { + pub(crate) token_address: String, + pub(crate) token_id: BigDecimal, +} diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 5af28c3f86..f52f16e0d7 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,4 +1,5 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftTxHistoryFilters, NftsTransferHistoryList}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTokenAddrId, NftTransferHistory, NftTxHistoryFilters, + NftsTransferHistoryList}; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; @@ -126,15 +127,19 @@ pub trait NftTxHistoryStorageOps { transaction_hash: String, ) -> MmResult, Self::Error>; - async fn update_tx_details_json_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error>; + async fn update_tx_meta_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error>; - async fn update_txs_coll_name_by_token_addr_id( + async fn update_txs_meta_by_token_addr_id( &self, chain: &Chain, token_address: String, token_id: BigDecimal, collection_name: Option, + image: Option, + token_name: Option, ) -> MmResult<(), Self::Error>; + + async fn get_txs_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error>; } #[derive(Debug, Deserialize, Display, Serialize)] diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 7ff2b86546..cba7b607e0 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,11 +1,11 @@ -use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTransferHistory, NftTxHistoryFilters, - NftsTransferHistoryList}; +use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTokenAddrId, NftTransferHistory, + NftTxHistoryFilters, NftsTransferHistoryList}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps, RemoveNftResult}; use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; -use db_common::sqlite::rusqlite::types::Type; +use db_common::sqlite::rusqlite::types::{FromSqlError, Type}; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; @@ -16,6 +16,7 @@ use mm2_number::BigDecimal; use serde_json::{self as json}; use std::convert::TryInto; use std::num::NonZeroUsize; +use std::str::FromStr; use std::sync::{Arc, Mutex}; fn nft_list_table_name(chain: &Chain) -> String { chain.to_ticker() + "_nft_list" } @@ -57,6 +58,9 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { token_id VARCHAR(256) NOT NULL, status TEXT NOT NULL, amount VARCHAR(256) NOT NULL, + collection_name TEXT, + image TEXT, + token_name TEXT, details_json TEXT );", table_name @@ -198,6 +202,16 @@ fn tx_history_from_row(row: &Row<'_>) -> Result { json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e))) } +fn token_address_id_from_row(row: &Row<'_>) -> Result { + let token_address: String = row.get("token_address")?; + let token_id_str: String = row.get("token_id")?; + let token_id = BigDecimal::from_str(&token_id_str).map_err(|_| SqlError::from(FromSqlError::InvalidType))?; + Ok(NftTokenAddrId { + token_address, + token_id, + }) +} + fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -220,9 +234,9 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT OR IGNORE INTO {} ( transaction_hash, chain, block_number, block_timestamp, contract_type, - token_address, token_id, status, amount, details_json + token_address, token_id, status, amount, collection_name, image, token_name, details_json ) VALUES ( - ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10 + ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13 );", table_name ); @@ -249,12 +263,12 @@ where Ok(sql) } -fn update_details_json_by_tx_hash_sql(chain: &Chain) -> MmResult { +fn update_meta_by_tx_hash_sql(chain: &Chain) -> MmResult { let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET details_json = ?1 WHERE transaction_hash = ?2;", + "UPDATE {} SET collection_name = ?1, image = ?2, token_name = ?3, details_json = ?4 WHERE transaction_hash = ?5;", table_name ); Ok(sql) @@ -387,6 +401,22 @@ fn get_txs_by_token_addr_id_builder<'a>( Ok(sql_builder) } +fn get_txs_with_empty_meta_builder<'a>(conn: &'a Connection, chain: &'a Chain) -> MmResult, SqlError> { + let table_name = nft_tx_history_table_name(chain); + validate_table_name(table_name.as_str())?; + let mut sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; + sql_builder + .sql_builder() + .distinct() + .field("token_address") + .field("token_id") + .and_where_is_null("collection_name") + .and_where_is_null("image") + .and_where_is_null("token_name"); + drop_mutability!(sql_builder); + Ok(sql_builder) +} + fn get_tx_by_tx_hash_sql(chain: &Chain) -> MmResult { let table_name = nft_tx_history_table_name(chain); validate_table_name(&table_name)?; @@ -735,6 +765,9 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.token_id.to_string()), Some(tx.status.to_string()), Some(tx.amount.to_string()), + tx.collection_name, + tx.image, + tx.token_name, Some(tx_json), ]; sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; @@ -802,15 +835,20 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn update_tx_details_json_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error> { - let sql = update_details_json_by_tx_hash_sql(chain)?; + async fn update_tx_meta_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error> { + let sql = update_meta_by_tx_hash_sql(chain)?; let tx_json = json::to_string(&tx).expect("serialization should not fail"); - let params = [tx_json, tx.transaction_hash]; + let params = [ + tx.collection_name, + tx.image, + tx.token_name, + Some(tx_json), + Some(tx.transaction_hash), + ]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&sql, ¶ms)?; sql_transaction.commit()?; Ok(()) @@ -818,20 +856,36 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn update_txs_coll_name_by_token_addr_id( + async fn update_txs_meta_by_token_addr_id( &self, chain: &Chain, token_address: String, token_id: BigDecimal, collection_name: Option, + image: Option, + token_name: Option, ) -> MmResult<(), Self::Error> { let selfi = self.clone(); let txs = selfi.get_txs_by_token_addr_id(chain, token_address, token_id).await?; for mut tx in txs.into_iter() { tx.collection_name = collection_name.clone(); + tx.image = image.clone(); + tx.token_name = token_name.clone(); drop_mutability!(tx); - selfi.update_tx_details_json_by_hash(chain, tx).await?; + selfi.update_tx_meta_by_hash(chain, tx).await?; } Ok(()) } + + async fn get_txs_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error> { + let selfi = self.clone(); + let chain = *chain; + async_blocking(move || { + let conn = selfi.0.lock().unwrap(); + let sql_builder = get_txs_with_empty_meta_builder(&conn, &chain)?; + let token_addr_id_pair = sql_builder.query(token_address_id_from_row)?; + Ok(token_addr_id_pair) + }) + .await + } } diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index c7754c60c4..d8f613b820 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -241,12 +241,14 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { todo!() } - async fn update_txs_coll_name_by_token_addr_id( + async fn update_txs_meta_by_token_addr_id( &self, _chain: &Chain, _token_address: String, _token_id: BigDecimal, _collection_name: Option, + _image: Option, + _token_name: Option, ) -> MmResult<(), Self::Error> { todo!() } From 94b9fe8d7fbb889f75008276071eb6a24c48f5ff Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 26 May 2023 19:26:32 +0700 Subject: [PATCH 049/112] fix conflicts --- mm2src/coins/nft.rs | 112 ++++++++++++++------------------ mm2src/coins/nft/nft_errors.rs | 5 +- mm2src/coins/nft/nft_structs.rs | 16 ----- 3 files changed, 50 insertions(+), 83 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 1384545e63..a2bfac13a3 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -1,7 +1,6 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::{MmError, MmResult}; use url::Url; -use std::str::FromStr; pub(crate) mod nft_errors; pub(crate) mod nft_structs; @@ -17,10 +16,8 @@ use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_structs::{TransferStatus, UriMeta}; use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::APPLICATION_JSON; -use ethereum_types::Address; use http::header::ACCEPT; use mm2_err_handle::map_to_mm::MapToMmResult; -use mm2_net::transport::SlurpError; use mm2_number::BigDecimal; use serde_json::Value as Json; use std::cmp::Ordering; @@ -293,27 +290,15 @@ async fn get_moralis_nft_transfers( // The cursor returned in the previous response (used for getting the next page). let mut cursor = String::new(); - let wallet_address = my_address.wallet_address;loop { + let wallet_address = my_address.wallet_address; + loop { let uri = format!("{}{}", uri_without_cursor, cursor); let response = send_request_to_uri(uri.as_str()).await?; if let Some(transfer_list) = response["result"].as_array() { for transfer in transfer_list { let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; - let status = if my_address.wallet_address.to_lowercase() == transfer_wrapper.to_address { - TransferStatus::Receive - } else { - TransferStatus::Send - }; let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); - let req = NftMetadataReq { - token_address: Address::from_str(&transfer_wrapper.token_address) - .map_to_mm(|e| GetNftInfoError::AddressError(e.to_string()))?, - token_id: transfer_wrapper.token_id.clone(), - chain, - url: req.url.clone(), - }; - let nft_meta = get_nft_metadata(ctx.clone(), req).await?; - lettransfer_history = NftTransferHistory { + let transfer_history = NftTransferHistory { chain: *chain, block_number: *transfer_wrapper.block_number, block_timestamp: transfer_wrapper.block_timestamp, @@ -329,9 +314,7 @@ async fn get_moralis_nft_transfers( collection_name: None, image: None, token_name: None, - collection_name: nft_meta.collection_name, - image: nft_meta.uri_meta.image, - token_name: nft_meta.uri_meta.token_name,from_address: transfer_wrapper.from_address, + from_address: transfer_wrapper.from_address, to_address: transfer_wrapper.to_address, status, amount: transfer_wrapper.amount.0, @@ -413,7 +396,6 @@ pub async fn withdraw_nft(ctx: MmArc, req: WithdrawNftReq) -> WithdrawNftResult } } - #[cfg(not(target_arch = "wasm32"))] async fn send_request_to_uri(uri: &str) -> MmResult { use http::header::HeaderValue; @@ -465,6 +447,28 @@ async fn send_request_to_uri(uri: &str) -> MmResult { Ok(response) } +async fn try_get_uri_meta(token_uri: &Option) -> MmResult { + match token_uri { + Some(token_uri) => { + if let Ok(response_meta) = send_request_to_uri(token_uri).await { + let uri_meta_res: UriMeta = serde_json::from_str(&response_meta.to_string())?; + Ok(uri_meta_res) + } else { + Ok(UriMeta::default()) + } + }, + None => Ok(UriMeta::default()), + } +} + +fn get_tx_status(my_wallet: &str, to_address: &str) -> TransferStatus { + // if my_wallet == from_address && my_wallet == to_address it is incoming tx, so we can check just to_address. + if my_wallet.to_lowercase() == to_address.to_lowercase() { + TransferStatus::Receive + } else { + TransferStatus::Send + } +} /// `update_nft_list` function gets nft transfers from NFT HISTORY table, iterates through them /// and updates NFT LIST table info. @@ -536,23 +540,16 @@ where if let Some(mut nft_db) = nft_db { match nft_db.amount.cmp(&tx.amount) { Ordering::Equal => { - if let Some(nft) = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) - .await? - { - storage - .update_txs_meta_by_token_addr_id( - chain, - nft.token_address, - nft.token_id, - nft.collection_name, - nft.uri_meta.image, - nft.uri_meta.token_name, - ) - .await?; - } else { - continue; - }; + storage + .update_txs_meta_by_token_addr_id( + chain, + nft_db.token_address, + nft_db.token_id, + nft_db.collection_name, + nft_db.uri_meta.image, + nft_db.uri_meta.token_name, + ) + .await?; storage .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) .await?; @@ -560,7 +557,19 @@ where Ordering::Greater => { nft_db.amount -= tx.amount; drop_mutability!(nft_db); - storage.update_nft_amount(chain, nft_db, tx.block_number).await?; + storage + .update_nft_amount(chain, nft_db.clone(), tx.block_number) + .await?; + storage + .update_txs_meta_by_token_addr_id( + chain, + nft_db.token_address, + nft_db.token_id, + nft_db.collection_name, + nft_db.uri_meta.image, + nft_db.uri_meta.token_name, + ) + .await?; }, Ordering::Less => { return MmError::err(UpdateNftError::InsufficientAmountInCache { @@ -733,26 +742,3 @@ where } Ok(()) } - -async fn try_get_uri_meta(token_uri: &Option) -> MmResult { - match token_uri { - Some(token_uri) => { - if let Ok(response_meta) = send_request_to_uri(token_uri).await { - let uri_meta_res: UriMeta = serde_json::from_str(&response_meta.to_string())?; - Ok(uri_meta_res) - } else { - Ok(UriMeta::default()) - } - }, - None => Ok(UriMeta::default()), - } -} - -fn get_tx_status(my_wallet: &str, to_address: &str) -> TransferStatus { - // if my_wallet == from_address && my_wallet == to_address it is incoming tx, so we can check just to_address. - if my_wallet.to_lowercase() == to_address.to_lowercase() { - TransferStatus::Receive - } else { - TransferStatus::Send - } -} diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index c386e97147..b3261381ee 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,5 +1,4 @@ use crate::eth::GetEthAddressError; -use crate::nft::GetInfoFromUriError; use crate::nft_storage::{CreateNftStorageError, NftStorageError}; use crate::GetMyAddressError; use common::HttpStatusCode; @@ -32,7 +31,6 @@ pub enum GetNftInfoError { token_address: String, token_id: String, }, - AddressError(String), #[display(fmt = "DB error {}", _0)] DbError(String), } @@ -100,8 +98,7 @@ impl HttpStatusCode for GetNftInfoError { | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) | GetNftInfoError::TokenNotFoundInWallet { .. } - | GetNftInfoError::DbError(_) - | GetNftInfoError::AddressError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index eb756e1c7b..3708aec138 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -131,16 +131,6 @@ pub(crate) struct UriMeta { animation_url: Option, } -#[derive(Debug, Default, Deserialize, Serialize)] -pub(crate) struct UriMeta { - pub(crate) image: Option, - #[serde(rename(deserialize = "name"))] - pub(crate) token_name: Option, - description: Option, - attributes: Option, - animation_url: Option, -} - #[derive(Clone, Debug, Deserialize, Serialize)] pub struct Nft { pub(crate) chain: Chain, @@ -315,12 +305,6 @@ impl fmt::Display for TransferStatus { } } -#[derive(Debug, Deserialize, Serialize)] -pub(crate) enum TransferStatus { - Receive, - Send, -} - #[derive(Debug, Deserialize, Serialize)] pub struct NftTransferHistory { pub(crate) chain: Chain, From b67e5e161953a0f0f38193a2ae4c04414b79759b Mon Sep 17 00:00:00 2001 From: laruh Date: Sat, 27 May 2023 00:06:17 +0700 Subject: [PATCH 050/112] add todos in wasm target --- mm2src/coins/nft/nft_structs.rs | 1 + mm2src/coins/nft_storage/wasm_storage.rs | 12 +++++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 3708aec138..bd4753a47b 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -361,6 +361,7 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } +#[allow(dead_code)] #[derive(Clone, Debug, Deserialize)] pub struct NftTxHistoryFilters { #[serde(default)] diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index d8f613b820..3f074828c7 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,6 +1,6 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryFilters, - NftTxHistoryStorageOps, RemoveNftResult}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTokenAddrId, + NftTxHistoryFilters, NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; use async_trait::async_trait; use derive_more::Display; @@ -233,11 +233,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { todo!() } - async fn update_tx_details_json_by_hash( - &self, - _chain: &Chain, - _tx: NftTransferHistory, - ) -> MmResult<(), Self::Error> { + async fn update_tx_meta_by_hash(&self, _chain: &Chain, _tx: NftTransferHistory) -> MmResult<(), Self::Error> { todo!() } @@ -252,4 +248,6 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { ) -> MmResult<(), Self::Error> { todo!() } + + async fn get_txs_with_empty_meta(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } } From 54220c3f7c350185035fc691316d60dead157843 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 28 May 2023 19:31:44 +0700 Subject: [PATCH 051/112] split update nft into fns, add TxMeta, add RefreshMetadataReq --- mm2src/coins/nft.rs | 421 ++++++++++++----------- mm2src/coins/nft/nft_structs.rs | 21 +- mm2src/coins/nft_storage/mod.rs | 12 +- mm2src/coins/nft_storage/sql_storage.rs | 22 +- mm2src/coins/nft_storage/wasm_storage.rs | 12 +- 5 files changed, 253 insertions(+), 235 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index a2bfac13a3..0dc5368036 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -13,7 +13,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, N TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::{TransferStatus, UriMeta}; +use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::APPLICATION_JSON; use http::header::ACCEPT; @@ -154,7 +154,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft Ok(()) } -pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<(), UpdateNftError> { +pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResult<(), UpdateNftError> { let moralis_meta = get_moralis_metadata( format!("{:#02x}", req.token_address), req.token_id.clone(), @@ -167,7 +167,6 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<( token_address: req.token_address, token_id: req.token_id, chain: req.chain, - url: req.url, }; let mut nft_db = get_nft_metadata(ctx, req).await?; let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; @@ -183,16 +182,14 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult<( storage .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) .await?; - storage - .update_txs_meta_by_token_addr_id( - &nft_db.chain, - nft_db.token_address, - nft_db.token_id, - nft_db.collection_name, - nft_db.uri_meta.image, - nft_db.uri_meta.token_name, - ) - .await?; + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(&nft_db.chain, tx_meta).await?; Ok(()) } @@ -472,185 +469,218 @@ fn get_tx_status(my_wallet: &str, to_address: &str) -> TransferStatus { /// `update_nft_list` function gets nft transfers from NFT HISTORY table, iterates through them /// and updates NFT LIST table info. -async fn update_nft_list( +async fn update_nft_list( ctx: MmArc, storage: &T, chain: &Chain, scan_from_block: u32, url: &Url, -) -> MmResult<(), UpdateNftError> -where - T: NftListStorageOps + NftTxHistoryStorageOps, -{ +) -> MmResult<(), UpdateNftError> { let txs = storage.get_txs_from_block(chain, scan_from_block).await?; + let req = MyAddressReq { + coin: chain.to_ticker(), + }; + let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); for tx in txs.into_iter() { - let req = MyAddressReq { - coin: chain.to_ticker(), + handle_nft_tx(storage, chain, url, tx, &my_address).await?; + } + Ok(()) +} + +async fn handle_nft_tx( + storage: &T, + chain: &Chain, + url: &Url, + tx: NftTransferHistory, + my_address: &str, +) -> MmResult<(), UpdateNftError> { + match (tx.status, tx.contract_type) { + (TransferStatus::Send, ContractType::Erc721) => handle_send_erc721(storage, chain, tx).await, + (TransferStatus::Receive, ContractType::Erc721) => { + handle_receive_erc721(storage, chain, tx, url, my_address).await + }, + (TransferStatus::Send, ContractType::Erc1155) => handle_send_erc1155(storage, chain, tx).await, + (TransferStatus::Receive, ContractType::Erc1155) => { + handle_receive_erc1155(storage, chain, tx, url, my_address).await + }, + } +} + +async fn handle_send_erc721( + storage: &T, + chain: &Chain, + tx: NftTransferHistory, +) -> MmResult<(), UpdateNftError> { + if let Some(nft_db) = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await? + { + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, }; - let my_address = get_my_address(ctx.clone(), req).await?.wallet_address.to_lowercase(); - match (tx.status, tx.contract_type) { - (TransferStatus::Send, ContractType::Erc721) => { - if let Some(nft) = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) - .await? - { - storage - .update_txs_meta_by_token_addr_id( - chain, - nft.token_address, - nft.token_id, - nft.collection_name, - nft.uri_meta.image, - nft.uri_meta.token_name, - ) - .await?; - } else { - continue; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .await?; + } else { + return MmError::err(UpdateNftError::TokenNotFoundInWallet { + token_address: tx.token_address, + token_id: tx.token_id.to_string(), + }); + } + Ok(()) +} + +async fn handle_receive_erc721( + storage: &T, + chain: &Chain, + tx: NftTransferHistory, + url: &Url, + my_address: &str, +) -> MmResult<(), UpdateNftError> { + let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; + // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later + // than History by Wallet update + nft.owner_of = my_address.to_string(); + nft.block_number = tx.block_number; + drop_mutability!(nft); + storage + .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number as u32) + .await?; + let tx_meta = TxMeta { + token_address: nft.token_address, + token_id: nft.token_id, + collection_name: nft.collection_name, + image: nft.uri_meta.image, + token_name: nft.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + Ok(()) +} + +async fn handle_send_erc1155( + storage: &T, + chain: &Chain, + tx: NftTransferHistory, +) -> MmResult<(), UpdateNftError> { + if let Some(mut nft_db) = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await? + { + match nft_db.amount.cmp(&tx.amount) { + Ordering::Equal => { + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; storage .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) .await?; }, - (TransferStatus::Receive, ContractType::Erc721) => { - let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; - // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later - // than History by Wallet update - nft.owner_of = my_address; - nft.block_number = tx.block_number; - drop_mutability!(nft); + Ordering::Greater => { + nft_db.amount -= tx.amount; + drop_mutability!(nft_db); storage - .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) - .await?; - storage - .update_txs_meta_by_token_addr_id( - chain, - nft.token_address, - nft.token_id, - nft.collection_name, - nft.uri_meta.image, - nft.uri_meta.token_name, - ) + .update_nft_amount(chain, nft_db.clone(), tx.block_number) .await?; + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; }, - (TransferStatus::Send, ContractType::Erc1155) => { - let nft_db = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) - .await?; - // If nft exists then check the amount - if let Some(mut nft_db) = nft_db { - match nft_db.amount.cmp(&tx.amount) { - Ordering::Equal => { - storage - .update_txs_meta_by_token_addr_id( - chain, - nft_db.token_address, - nft_db.token_id, - nft_db.collection_name, - nft_db.uri_meta.image, - nft_db.uri_meta.token_name, - ) - .await?; - storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) - .await?; - }, - Ordering::Greater => { - nft_db.amount -= tx.amount; - drop_mutability!(nft_db); - storage - .update_nft_amount(chain, nft_db.clone(), tx.block_number) - .await?; - storage - .update_txs_meta_by_token_addr_id( - chain, - nft_db.token_address, - nft_db.token_id, - nft_db.collection_name, - nft_db.uri_meta.image, - nft_db.uri_meta.token_name, - ) - .await?; - }, - Ordering::Less => { - return MmError::err(UpdateNftError::InsufficientAmountInCache { - amount_list: nft_db.amount.to_string(), - amount_history: tx.amount.to_string(), - }); - }, - } - } else { - // token must exist in NFT LIST table - return MmError::err(UpdateNftError::TokenNotFoundInWallet { - token_address: tx.token_address, - token_id: tx.token_id.to_string(), - }); - } - }, - (TransferStatus::Receive, ContractType::Erc1155) => { - let nft_db = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) - .await?; - // If token isn't in NFT LIST table then add nft to the table. - if let Some(mut nft_db) = nft_db { - // if owner address == from address, then owner sent tokens to themself, - // which means that the amount will not change. - if my_address != tx.from_address { - nft_db.amount += tx.amount; - } - nft_db.block_number = tx.block_number; - drop_mutability!(nft_db); - storage - .update_nft_amount_and_block_number(chain, nft_db.clone()) - .await?; - storage - .update_txs_meta_by_token_addr_id( - chain, - nft_db.token_address, - nft_db.token_id, - nft_db.collection_name, - nft_db.uri_meta.image, - nft_db.uri_meta.token_name, - ) - .await?; - } else { - let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; - let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; - let nft = Nft { - chain: *chain, - token_address: moralis_meta.token_address, - token_id: moralis_meta.token_id, - amount: tx.amount, - owner_of: my_address, - token_hash: moralis_meta.token_hash, - block_number_minted: moralis_meta.block_number_minted, - block_number: tx.block_number, - contract_type: moralis_meta.contract_type, - collection_name: moralis_meta.collection_name, - symbol: moralis_meta.symbol, - token_uri: moralis_meta.token_uri, - metadata: moralis_meta.metadata, - last_token_uri_sync: moralis_meta.last_token_uri_sync, - last_metadata_sync: moralis_meta.last_metadata_sync, - minter_address: moralis_meta.minter_address, - possible_spam: moralis_meta.possible_spam, - uri_meta, - }; - storage - .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) - .await?; - storage - .update_txs_meta_by_token_addr_id( - chain, - nft.token_address, - nft.token_id, - nft.collection_name, - nft.uri_meta.image, - nft.uri_meta.token_name, - ) - .await?; - } + Ordering::Less => { + return MmError::err(UpdateNftError::InsufficientAmountInCache { + amount_list: nft_db.amount.to_string(), + amount_history: tx.amount.to_string(), + }); }, } + } else { + // token must exist in NFT LIST table + return MmError::err(UpdateNftError::TokenNotFoundInWallet { + token_address: tx.token_address, + token_id: tx.token_id.to_string(), + }); + } + Ok(()) +} + +async fn handle_receive_erc1155( + storage: &T, + chain: &Chain, + tx: NftTransferHistory, + url: &Url, + my_address: &str, +) -> MmResult<(), UpdateNftError> { + // If token isn't in NFT LIST table then add nft to the table. + if let Some(mut nft_db) = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await? + { + // if owner address == from address, then owner sent tokens to themself, + // which means that the amount will not change. + if my_address != tx.from_address { + nft_db.amount += tx.amount; + } + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage + .update_nft_amount_and_block_number(chain, nft_db.clone()) + .await?; + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + } else { + let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; + let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; + let nft = Nft { + chain: *chain, + token_address: moralis_meta.token_address, + token_id: moralis_meta.token_id, + amount: tx.amount, + owner_of: my_address.to_string(), + token_hash: moralis_meta.token_hash, + block_number_minted: moralis_meta.block_number_minted, + block_number: tx.block_number, + contract_type: moralis_meta.contract_type, + collection_name: moralis_meta.collection_name, + symbol: moralis_meta.symbol, + token_uri: moralis_meta.token_uri, + metadata: moralis_meta.metadata, + last_token_uri_sync: moralis_meta.last_token_uri_sync, + last_metadata_sync: moralis_meta.last_metadata_sync, + minter_address: moralis_meta.minter_address, + possible_spam: moralis_meta.possible_spam, + uri_meta, + }; + storage + .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) + .await?; + let tx_meta = TxMeta { + token_address: nft.token_address, + token_id: nft.token_id, + collection_name: nft.collection_name, + image: nft.uri_meta.image, + token_name: nft.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } Ok(()) } @@ -684,15 +714,12 @@ pub(crate) async fn find_wallet_nft_amount( Ok(wallet_amount) } -async fn cache_nfts_from_moralis( +async fn cache_nfts_from_moralis( ctx: &MmArc, storage: &T, chain: &Chain, url: &Url, -) -> MmResult, UpdateNftError> -where - T: NftListStorageOps + NftTxHistoryStorageOps, -{ +) -> MmResult, UpdateNftError> { let nft_list = get_moralis_nft_list(ctx, chain, url).await?; let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(storage, chain) .await? @@ -708,16 +735,14 @@ where T: NftListStorageOps + NftTxHistoryStorageOps, { for nft in nfts.into_iter() { - storage - .update_txs_meta_by_token_addr_id( - chain, - nft.token_address, - nft.token_id, - nft.collection_name, - nft.uri_meta.image, - nft.uri_meta.token_name, - ) - .await?; + let tx_meta = TxMeta { + token_address: nft.token_address, + token_id: nft.token_id, + collection_name: nft.collection_name, + image: nft.uri_meta.image, + token_name: nft.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } Ok(()) } @@ -729,16 +754,14 @@ where let nft_token_addr_id = storage.get_txs_with_empty_meta(chain).await?; for addr_id_pair in nft_token_addr_id.into_iter() { let nft_meta = get_moralis_metadata(addr_id_pair.token_address, addr_id_pair.token_id, chain, url).await?; - storage - .update_txs_meta_by_token_addr_id( - chain, - nft_meta.token_address, - nft_meta.token_id, - nft_meta.collection_name, - nft_meta.uri_meta.image, - nft_meta.uri_meta.token_name, - ) - .await?; + let tx_meta = TxMeta { + token_address: nft_meta.token_address, + token_id: nft_meta.token_id, + collection_name: nft_meta.collection_name, + image: nft_meta.uri_meta.image, + token_name: nft_meta.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } Ok(()) } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index bd4753a47b..59910bd58f 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -25,6 +25,13 @@ pub struct NftMetadataReq { pub(crate) token_address: Address, pub(crate) token_id: BigDecimal, pub(crate) chain: Chain, +} + +#[derive(Debug, Deserialize)] +pub struct RefreshMetadataReq { + pub(crate) token_address: Address, + pub(crate) token_id: BigDecimal, + pub(crate) chain: Chain, pub(crate) url: Url, } @@ -92,7 +99,7 @@ pub(crate) enum ParseContractTypeError { UnsupportedContractType, } -#[derive(Clone, Debug, Deserialize, Serialize)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "UPPERCASE")] pub(crate) enum ContractType { Erc1155, @@ -277,7 +284,7 @@ pub(crate) enum ParseTransferStatusError { UnsupportedTransferStatus, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Clone, Copy, Serialize)] pub(crate) enum TransferStatus { Receive, Send, @@ -383,3 +390,13 @@ pub struct NftTokenAddrId { pub(crate) token_address: String, pub(crate) token_id: BigDecimal, } + +#[allow(dead_code)] +#[derive(Debug)] +pub struct TxMeta { + pub(crate) token_address: String, + pub(crate) token_id: BigDecimal, + pub(crate) collection_name: Option, + pub(crate) image: Option, + pub(crate) token_name: Option, +} diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index f52f16e0d7..1e9cf7260f 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -1,5 +1,5 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTokenAddrId, NftTransferHistory, NftTxHistoryFilters, - NftsTransferHistoryList}; + NftsTransferHistoryList, TxMeta}; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; @@ -129,15 +129,7 @@ pub trait NftTxHistoryStorageOps { async fn update_tx_meta_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error>; - async fn update_txs_meta_by_token_addr_id( - &self, - chain: &Chain, - token_address: String, - token_id: BigDecimal, - collection_name: Option, - image: Option, - token_name: Option, - ) -> MmResult<(), Self::Error>; + async fn update_txs_meta_by_token_addr_id(&self, chain: &Chain, tx_meta: TxMeta) -> MmResult<(), Self::Error>; async fn get_txs_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error>; } diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index cba7b607e0..49bd97d9c9 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -1,5 +1,5 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTokenAddrId, NftTransferHistory, - NftTxHistoryFilters, NftsTransferHistoryList}; + NftTxHistoryFilters, NftsTransferHistoryList, TxMeta}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps, RemoveNftResult}; use async_trait::async_trait; @@ -856,21 +856,15 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn update_txs_meta_by_token_addr_id( - &self, - chain: &Chain, - token_address: String, - token_id: BigDecimal, - collection_name: Option, - image: Option, - token_name: Option, - ) -> MmResult<(), Self::Error> { + async fn update_txs_meta_by_token_addr_id(&self, chain: &Chain, tx_meta: TxMeta) -> MmResult<(), Self::Error> { let selfi = self.clone(); - let txs = selfi.get_txs_by_token_addr_id(chain, token_address, token_id).await?; + let txs = selfi + .get_txs_by_token_addr_id(chain, tx_meta.token_address, tx_meta.token_id) + .await?; for mut tx in txs.into_iter() { - tx.collection_name = collection_name.clone(); - tx.image = image.clone(); - tx.token_name = token_name.clone(); + tx.collection_name = tx_meta.collection_name.clone(); + tx.image = tx_meta.image.clone(); + tx.token_name = tx_meta.token_name.clone(); drop_mutability!(tx); selfi.update_tx_meta_by_hash(chain, tx).await?; } diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm_storage.rs index 3f074828c7..da714b3503 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm_storage.rs @@ -1,4 +1,4 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList}; +use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList, TxMeta}; use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTokenAddrId, NftTxHistoryFilters, NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; @@ -237,15 +237,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { todo!() } - async fn update_txs_meta_by_token_addr_id( - &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, - _collection_name: Option, - _image: Option, - _token_name: Option, - ) -> MmResult<(), Self::Error> { + async fn update_txs_meta_by_token_addr_id(&self, _chain: &Chain, _tx_meta: TxMeta) -> MmResult<(), Self::Error> { todo!() } From e3d19f107a3d711dfd95ca9ffd60f3b9712fd743 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 29 May 2023 00:51:31 +0700 Subject: [PATCH 052/112] send and receive filters fix --- mm2src/coins/nft_storage/sql_storage.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 49bd97d9c9..6dc2af1ec4 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -149,9 +149,9 @@ fn nft_history_table_builder_preimage<'a>( let mut sql_builder = SqlQuery::select_from(conn, table_name)?; if let Some(filters) = filters { if filters.send && !filters.receive { - sql_builder.sql_builder().and_where_eq("status", "Send"); + sql_builder.sql_builder().and_where_eq("status", "'Send'"); } else if filters.receive && !filters.send { - sql_builder.sql_builder().and_where_eq("status", "Receive"); + sql_builder.sql_builder().and_where_eq("status", "'Receive'"); } if filters.from_date.is_some() && filters.to_date.is_some() { From be984ef02259e9f32420482db4951de098c9fbfb Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 31 May 2023 21:43:40 +0700 Subject: [PATCH 053/112] move nft wasm db into wasm module, add tables into wasm --- mm2src/coins/lp_coins.rs | 2 +- mm2src/coins/nft_storage/mod.rs | 4 +- mm2src/coins/nft_storage/wasm/mod.rs | 59 +++++++ mm2src/coins/nft_storage/wasm/nft_idb.rs | 32 ++++ .../nft_storage/{ => wasm}/wasm_storage.rs | 158 ++++++++---------- 5 files changed, 168 insertions(+), 87 deletions(-) create mode 100644 mm2src/coins/nft_storage/wasm/mod.rs create mode 100644 mm2src/coins/nft_storage/wasm/nft_idb.rs rename mm2src/coins/nft_storage/{ => wasm}/wasm_storage.rs (59%) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 70b9ceb632..a7214745a4 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -100,7 +100,7 @@ cfg_wasm32! { use mm2_db::indexed_db::{ConstructibleDb, DbLocked, SharedDb}; use tx_history_storage::wasm::{clear_tx_history, load_tx_history, save_tx_history, TxHistoryDb}; pub type TxHistoryDbLocked<'a> = DbLocked<'a, TxHistoryDb>; - use nft_storage::wasm_storage::NftCacheIDB; + use nft_storage::wasm::nft_idb::NftCacheIDB; } // using custom copy of try_fus as futures crate was renamed to futures01 diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 1e9cf7260f..0380b0cdda 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize}; use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; -#[cfg(target_arch = "wasm32")] pub mod wasm_storage; +#[cfg(target_arch = "wasm32")] pub mod wasm; #[derive(Debug)] pub enum RemoveNftResult { @@ -152,7 +152,7 @@ impl<'a> NftStorageBuilder<'a> { #[inline] pub fn build(self) -> MmResult { #[cfg(target_arch = "wasm32")] - return wasm_storage::IndexedDbNftStorage::new(self.ctx); + return wasm::wasm_storage::IndexedDbNftStorage::new(self.ctx); #[cfg(not(target_arch = "wasm32"))] sql_storage::SqliteNftStorage::new(self.ctx) } diff --git a/mm2src/coins/nft_storage/wasm/mod.rs b/mm2src/coins/nft_storage/wasm/mod.rs new file mode 100644 index 0000000000..47168452d7 --- /dev/null +++ b/mm2src/coins/nft_storage/wasm/mod.rs @@ -0,0 +1,59 @@ +use crate::nft_storage::NftStorageError; +use mm2_db::indexed_db::{DbTransactionError, InitDbError}; +use mm2_err_handle::prelude::*; + +pub mod nft_idb; +pub mod wasm_storage; + +pub type WasmNftCacheResult = MmResult; + +impl NftStorageError for WasmNftCacheError {} + +#[derive(Debug, Display)] +pub enum WasmNftCacheError { + ErrorSerializing(String), + ErrorDeserializing(String), + ErrorSaving(String), + ErrorLoading(String), + ErrorClearing(String), + NotSupported(String), + InternalError(String), +} + +impl From for WasmNftCacheError { + fn from(e: InitDbError) -> Self { + match &e { + InitDbError::NotSupported(_) => WasmNftCacheError::NotSupported(e.to_string()), + InitDbError::EmptyTableList + | InitDbError::DbIsOpenAlready { .. } + | InitDbError::InvalidVersion(_) + | InitDbError::OpeningError(_) + | InitDbError::TypeMismatch { .. } + | InitDbError::UnexpectedState(_) + | InitDbError::UpgradingError { .. } => WasmNftCacheError::InternalError(e.to_string()), + } + } +} + +impl From for WasmNftCacheError { + fn from(e: DbTransactionError) -> Self { + match e { + DbTransactionError::ErrorSerializingItem(_) => WasmNftCacheError::ErrorSerializing(e.to_string()), + DbTransactionError::ErrorDeserializingItem(_) => WasmNftCacheError::ErrorDeserializing(e.to_string()), + DbTransactionError::ErrorUploadingItem(_) => WasmNftCacheError::ErrorSaving(e.to_string()), + DbTransactionError::ErrorGettingItems(_) | DbTransactionError::ErrorCountingItems(_) => { + WasmNftCacheError::ErrorLoading(e.to_string()) + }, + DbTransactionError::ErrorDeletingItems(_) => WasmNftCacheError::ErrorClearing(e.to_string()), + DbTransactionError::NoSuchTable { .. } + | DbTransactionError::ErrorCreatingTransaction(_) + | DbTransactionError::ErrorOpeningTable { .. } + | DbTransactionError::ErrorSerializingIndex { .. } + | DbTransactionError::UnexpectedState(_) + | DbTransactionError::TransactionAborted + | DbTransactionError::MultipleItemsByUniqueIndex { .. } + | DbTransactionError::NoSuchIndex { .. } + | DbTransactionError::InvalidIndex { .. } => WasmNftCacheError::InternalError(e.to_string()), + } + } +} diff --git a/mm2src/coins/nft_storage/wasm/nft_idb.rs b/mm2src/coins/nft_storage/wasm/nft_idb.rs new file mode 100644 index 0000000000..8842cf9b3c --- /dev/null +++ b/mm2src/coins/nft_storage/wasm/nft_idb.rs @@ -0,0 +1,32 @@ +use crate::nft_storage::wasm::wasm_storage::{NftListTable, NftTxHistoryTable}; +use async_trait::async_trait; +use mm2_db::indexed_db::InitDbResult; +use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder}; + +const DB_NAME: &str = "nft_cache"; +const DB_VERSION: u32 = 1; +pub type NftCacheIDBLocked<'a> = DbLocked<'a, NftCacheIDB>; + +pub struct NftCacheIDB { + inner: IndexedDb, +} + +#[async_trait] +impl DbInstance for NftCacheIDB { + fn db_name() -> &'static str { DB_NAME } + + async fn init(db_id: DbIdentifier) -> InitDbResult { + let inner = IndexedDbBuilder::new(db_id) + .with_version(DB_VERSION) + .with_table::() + .with_table::() + .build() + .await?; + Ok(NftCacheIDB { inner }) + } +} + +#[allow(dead_code)] +impl NftCacheIDB { + fn get_inner(&self) -> &IndexedDb { &self.inner } +} diff --git a/mm2src/coins/nft_storage/wasm_storage.rs b/mm2src/coins/nft_storage/wasm/wasm_storage.rs similarity index 59% rename from mm2src/coins/nft_storage/wasm_storage.rs rename to mm2src/coins/nft_storage/wasm/wasm_storage.rs index da714b3503..d60908d647 100644 --- a/mm2src/coins/nft_storage/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm/wasm_storage.rs @@ -1,96 +1,20 @@ -use crate::nft::nft_structs::{Chain, Nft, NftList, NftTransferHistory, NftsTransferHistoryList, TxMeta}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTokenAddrId, - NftTxHistoryFilters, NftTxHistoryStorageOps, RemoveNftResult}; +use crate::nft::nft_structs::{Chain, ContractType, Nft, NftList, NftTransferHistory, NftsTransferHistoryList, + TransferStatus, TxMeta}; +use crate::nft_storage::wasm::nft_idb::{NftCacheIDB, NftCacheIDBLocked}; +use crate::nft_storage::wasm::{WasmNftCacheError, WasmNftCacheResult}; +use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTokenAddrId, NftTxHistoryFilters, + NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; use async_trait::async_trait; -use derive_more::Display; use mm2_core::mm_ctx::MmArc; -pub use mm2_db::indexed_db::InitDbResult; -use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, DbTransactionError, IndexedDb, IndexedDbBuilder, - InitDbError, SharedDb}; +use mm2_db::indexed_db::{BeBigUint, DbUpgrader, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; use mm2_number::BigDecimal; +use serde_json::Value as Json; use std::num::NonZeroUsize; -const DB_NAME: &str = "nft_cache"; -const DB_VERSION: u32 = 1; - -pub type WasmNftCacheResult = MmResult; -pub type NftCacheIDBLocked<'a> = DbLocked<'a, NftCacheIDB>; - -impl NftStorageError for WasmNftCacheError {} - -#[derive(Debug, Display)] -pub enum WasmNftCacheError { - ErrorSerializing(String), - ErrorDeserializing(String), - ErrorSaving(String), - ErrorLoading(String), - ErrorClearing(String), - NotSupported(String), - InternalError(String), -} - -impl From for WasmNftCacheError { - fn from(e: InitDbError) -> Self { - match &e { - InitDbError::NotSupported(_) => WasmNftCacheError::NotSupported(e.to_string()), - InitDbError::EmptyTableList - | InitDbError::DbIsOpenAlready { .. } - | InitDbError::InvalidVersion(_) - | InitDbError::OpeningError(_) - | InitDbError::TypeMismatch { .. } - | InitDbError::UnexpectedState(_) - | InitDbError::UpgradingError { .. } => WasmNftCacheError::InternalError(e.to_string()), - } - } -} - -impl From for WasmNftCacheError { - fn from(e: DbTransactionError) -> Self { - match e { - DbTransactionError::ErrorSerializingItem(_) => WasmNftCacheError::ErrorSerializing(e.to_string()), - DbTransactionError::ErrorDeserializingItem(_) => WasmNftCacheError::ErrorDeserializing(e.to_string()), - DbTransactionError::ErrorUploadingItem(_) => WasmNftCacheError::ErrorSaving(e.to_string()), - DbTransactionError::ErrorGettingItems(_) | DbTransactionError::ErrorCountingItems(_) => { - WasmNftCacheError::ErrorLoading(e.to_string()) - }, - DbTransactionError::ErrorDeletingItems(_) => WasmNftCacheError::ErrorClearing(e.to_string()), - DbTransactionError::NoSuchTable { .. } - | DbTransactionError::ErrorCreatingTransaction(_) - | DbTransactionError::ErrorOpeningTable { .. } - | DbTransactionError::ErrorSerializingIndex { .. } - | DbTransactionError::UnexpectedState(_) - | DbTransactionError::TransactionAborted - | DbTransactionError::MultipleItemsByUniqueIndex { .. } - | DbTransactionError::NoSuchIndex { .. } - | DbTransactionError::InvalidIndex { .. } => WasmNftCacheError::InternalError(e.to_string()), - } - } -} - -pub struct NftCacheIDB { - inner: IndexedDb, -} - -#[async_trait] -impl DbInstance for NftCacheIDB { - fn db_name() -> &'static str { DB_NAME } - - async fn init(db_id: DbIdentifier) -> InitDbResult { - // todo add tables for each chain - let inner = IndexedDbBuilder::new(db_id).with_version(DB_VERSION).build().await?; - Ok(NftCacheIDB { inner }) - } -} - -#[allow(dead_code)] -impl NftCacheIDB { - fn get_inner(&self) -> &IndexedDb { &self.inner } -} - #[derive(Clone)] pub struct IndexedDbNftStorage { db: SharedDb, @@ -243,3 +167,69 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn get_txs_with_empty_meta(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } } + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct NftListTable { + token_address: String, + token_id: BeBigUint, + chain: String, + amount: BeBigUint, + block_number: u32, + contract_type: ContractType, + details_json: Json, +} + +impl NftListTable { + pub const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; +} + +impl TableSignature for NftListTable { + fn table_name() -> &'static str { "nft_list_cache_table" } + + fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { + if let (0, 1) = (old_version, new_version) { + let table = upgrader.create_table(Self::table_name())?; + table.create_multi_index( + Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, + &["chain", "token_address", "token_id"], + true, + )?; + table.create_index("chain", false)?; + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct NftTxHistoryTable { + transaction_hash: String, + chain: String, + block_number: u32, + block_timestamp: String, + contract_type: ContractType, + token_address: String, + token_id: BeBigUint, + status: TransferStatus, + amount: BeBigUint, + collection_name: String, + image: String, + token_name: String, + details_json: Json, +} + +impl NftTxHistoryTable { + pub const CHAIN_TX_HASH_INDEX: &str = "chain_tx_hash_index"; +} + +impl TableSignature for NftTxHistoryTable { + fn table_name() -> &'static str { "nft_tx_history_cache_table" } + + fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { + if let (0, 1) = (old_version, new_version) { + let table = upgrader.create_table(Self::table_name())?; + table.create_multi_index(Self::CHAIN_TX_HASH_INDEX, &["chain", "transaction_hash"], true)?; + table.create_index("chain", false)?; + } + Ok(()) + } +} From 38814e4a5d2267f114b08d530b8445b221308aeb Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 1 Jun 2023 15:00:34 +0700 Subject: [PATCH 054/112] use ok_or_else when get_nft() --- mm2src/coins/nft.rs | 149 ++++++++++++++++++++------------------------ 1 file changed, 66 insertions(+), 83 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 0dc5368036..b18bc75134 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -29,7 +29,7 @@ const MORALIS_FORMAT_QUERY_VALUE: &str = "decimal"; /// query parameters for moralis request: The transfer direction const MORALIS_DIRECTION_QUERY_NAME: &str = "direction"; const MORALIS_DIRECTION_QUERY_VALUE: &str = "both"; -// The minimum block number from which to get the transfers +/// The minimum block number from which to get the transfers const MORALIS_FROM_BLOCK_QUERY_NAME: &str = "from_block"; pub type WithdrawNftResult = Result>; @@ -56,15 +56,12 @@ pub async fn get_nft_metadata(ctx: MmArc, req: NftMetadataReq) -> MmResult MmResult<(), UpdateNft } }, // If the last scanned block value is absent, we cannot accurately update the NFT cache. - // This is because a situation may occur where a user doesn't transfer all ERC-1155 tokens, + // This is because a situation may occur where the user doesn't transfer all ERC-1155 tokens, // resulting in the block number of NFT remaining unchanged. (None, Some(nft_block)) => { return MmError::err(UpdateNftError::LastScannedBlockNotFound { @@ -511,27 +508,24 @@ async fn handle_send_erc721( chain: &Chain, tx: NftTransferHistory, ) -> MmResult<(), UpdateNftError> { - if let Some(nft_db) = storage + let nft_db = storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await? - { - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) - .await?; - } else { - return MmError::err(UpdateNftError::TokenNotFoundInWallet { - token_address: tx.token_address, + .ok_or_else(|| UpdateNftError::TokenNotFoundInWallet { + token_address: tx.token_address.clone(), token_id: tx.token_id.to_string(), - }); - } + })?; + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .await?; Ok(()) } @@ -567,52 +561,48 @@ async fn handle_send_erc1155( chain: &Chain, tx: NftTransferHistory, ) -> MmResult<(), UpdateNftError> { - if let Some(mut nft_db) = storage + let mut nft_db = storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await? - { - match nft_db.amount.cmp(&tx.amount) { - Ordering::Equal => { - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) - .await?; - }, - Ordering::Greater => { - nft_db.amount -= tx.amount; - drop_mutability!(nft_db); - storage - .update_nft_amount(chain, nft_db.clone(), tx.block_number) - .await?; - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - }, - Ordering::Less => { - return MmError::err(UpdateNftError::InsufficientAmountInCache { - amount_list: nft_db.amount.to_string(), - amount_history: tx.amount.to_string(), - }); - }, - } - } else { - // token must exist in NFT LIST table - return MmError::err(UpdateNftError::TokenNotFoundInWallet { - token_address: tx.token_address, + .ok_or_else(|| UpdateNftError::TokenNotFoundInWallet { + token_address: tx.token_address.clone(), token_id: tx.token_id.to_string(), - }); + })?; + match nft_db.amount.cmp(&tx.amount) { + Ordering::Equal => { + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + storage + .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .await?; + }, + Ordering::Greater => { + nft_db.amount -= tx.amount; + drop_mutability!(nft_db); + storage + .update_nft_amount(chain, nft_db.clone(), tx.block_number) + .await?; + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + collection_name: nft_db.collection_name, + image: nft_db.uri_meta.image, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + }, + Ordering::Less => { + return MmError::err(UpdateNftError::InsufficientAmountInCache { + amount_list: nft_db.amount.to_string(), + amount_history: tx.amount.to_string(), + }); + }, } Ok(()) } @@ -697,21 +687,14 @@ pub(crate) async fn find_wallet_nft_amount( if !NftListStorageOps::is_initialized(&storage, chain).await? { NftListStorageOps::init(&storage, chain).await?; } - let nft_meta = storage .get_nft(chain, token_address.to_lowercase(), token_id.clone()) - .await?; - - let wallet_amount = match nft_meta { - Some(nft) => nft.amount, - None => { - return MmError::err(GetNftInfoError::TokenNotFoundInWallet { - token_address, - token_id: token_id.to_string(), - }) - }, - }; - Ok(wallet_amount) + .await? + .ok_or_else(|| GetNftInfoError::TokenNotFoundInWallet { + token_address, + token_id: token_id.to_string(), + })?; + Ok(nft_meta.amount) } async fn cache_nfts_from_moralis( From f919a379af67670bde917a03bd7de2e2ae632c50 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 1 Jun 2023 16:18:53 +0700 Subject: [PATCH 055/112] use pub(crate) for mod nft_storage, move import below it --- mm2src/coins/lp_coins.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index a7214745a4..9d05979104 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -288,11 +288,11 @@ use utxo::{BlockchainNetwork, GenerateTxError, UtxoFeeDetails, UtxoTx}; pub mod nft; use nft::nft_errors::GetNftInfoError; -pub mod nft_storage; +pub(crate) mod nft_storage; +use crate::nft_storage::{CreateNftStorageError, NftStorageError}; #[cfg(not(target_arch = "wasm32"))] pub mod z_coin; -use crate::nft_storage::{CreateNftStorageError, NftStorageError}; #[cfg(not(target_arch = "wasm32"))] use z_coin::ZCoin; pub type TransactionFut = Box + Send>; From 217c1997a907608764fb9049892bb0180b724c1e Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 1 Jun 2023 16:27:05 +0700 Subject: [PATCH 056/112] allow(dead_code) for RemoveNftResult --- mm2src/coins/nft_storage/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 0380b0cdda..13d7f388af 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -12,6 +12,7 @@ use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm; +#[allow(dead_code)] #[derive(Debug)] pub enum RemoveNftResult { NftRemoved, From bbe4be5872de301cdba66bf40f2165fdd693e32c Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 1 Jun 2023 17:52:12 +0700 Subject: [PATCH 057/112] add LastScannedBlockTable --- mm2src/coins/nft_storage/wasm/nft_idb.rs | 3 +- mm2src/coins/nft_storage/wasm/wasm_storage.rs | 38 ++++++++++++++++--- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/nft_storage/wasm/nft_idb.rs b/mm2src/coins/nft_storage/wasm/nft_idb.rs index 8842cf9b3c..89c17ade32 100644 --- a/mm2src/coins/nft_storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft_storage/wasm/nft_idb.rs @@ -1,4 +1,4 @@ -use crate::nft_storage::wasm::wasm_storage::{NftListTable, NftTxHistoryTable}; +use crate::nft_storage::wasm::wasm_storage::{LastScannedBlockTable, NftListTable, NftTxHistoryTable}; use async_trait::async_trait; use mm2_db::indexed_db::InitDbResult; use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder}; @@ -20,6 +20,7 @@ impl DbInstance for NftCacheIDB { .with_version(DB_VERSION) .with_table::() .with_table::() + .with_table::() .build() .await?; Ok(NftCacheIDB { inner }) diff --git a/mm2src/coins/nft_storage/wasm/wasm_storage.rs b/mm2src/coins/nft_storage/wasm/wasm_storage.rs index d60908d647..70dc95c380 100644 --- a/mm2src/coins/nft_storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm/wasm_storage.rs @@ -12,7 +12,7 @@ use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; use mm2_number::BigDecimal; -use serde_json::Value as Json; +use serde_json::{self as json, Value as Json}; use std::num::NonZeroUsize; #[derive(Clone)] @@ -38,9 +38,9 @@ impl IndexedDbNftStorage { impl NftListStorageOps for IndexedDbNftStorage { type Error = WasmNftCacheError; - async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { Ok(()) } - async fn is_initialized(&self, _chain: &Chain) -> MmResult { todo!() } + async fn is_initialized(&self, _chain: &Chain) -> MmResult { Ok(true) } async fn get_nft_list( &self, @@ -107,9 +107,9 @@ impl NftListStorageOps for IndexedDbNftStorage { impl NftTxHistoryStorageOps for IndexedDbNftStorage { type Error = WasmNftCacheError; - async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { todo!() } + async fn init(&self, _chain: &Chain) -> MmResult<(), Self::Error> { Ok(()) } - async fn is_initialized(&self, _chain: &Chain) -> MmResult { todo!() } + async fn is_initialized(&self, _chain: &Chain) -> MmResult { Ok(true) } async fn get_tx_history( &self, @@ -200,6 +200,11 @@ impl TableSignature for NftListTable { } } +#[allow(dead_code)] +fn nft_details_from_item(item: NftListTable) -> WasmNftCacheResult { + json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct NftTxHistoryTable { transaction_hash: String, @@ -233,3 +238,26 @@ impl TableSignature for NftTxHistoryTable { Ok(()) } } + +#[allow(dead_code)] +fn tx_details_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult { + json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub(crate) struct LastScannedBlockTable { + chain: String, + last_scanned_block: u32, +} + +impl TableSignature for LastScannedBlockTable { + fn table_name() -> &'static str { "last_scanned_block_table" } + + fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { + if let (0, 1) = (old_version, new_version) { + let table = upgrader.create_table(Self::table_name())?; + table.create_index("chain", true)?; + } + Ok(()) + } +} From 9c1e2f4d83dcfa39cd8efa97a863c621ea31dadb Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 4 Jun 2023 19:17:07 +0700 Subject: [PATCH 058/112] error in union_sql_strings, Some for date filters,remove nft_table_builder_preimage, use INSERT INTO, upsert func --- mm2src/coins/nft_storage/sql_storage.rs | 77 +++++++++---------------- 1 file changed, 26 insertions(+), 51 deletions(-) diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 6dc2af1ec4..58ccf4cec8 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -98,18 +98,16 @@ impl SqliteNftStorage { } fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmResult { - let union_sql_strings: MmResult, SqlError> = chains + let union_sql_strings = chains .iter() .map(|chain| { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; - let sql_builder = nft_table_builder_preimage(conn, table_name.as_str())?; + let sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); Ok(sql_string) }) - .collect(); - - let union_sql_strings = union_sql_strings?; + .collect::, SqlError>>()?; let union_sql = union_sql_strings.join(" UNION ALL "); let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; final_sql_builder.order_desc("nft_list.block_number")?; @@ -122,7 +120,7 @@ fn get_nft_tx_builder_preimage( chains: Vec, filters: Option, ) -> MmResult { - let union_sql_strings: MmResult, SqlError> = chains + let union_sql_strings = chains .into_iter() .map(|chain| { let table_name = nft_tx_history_table_name(&chain); @@ -131,9 +129,7 @@ fn get_nft_tx_builder_preimage( let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); Ok(sql_string) }) - .collect(); - - let union_sql_strings = union_sql_strings?; + .collect::, SqlError>>()?; let union_sql = union_sql_strings.join(" UNION ALL "); let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_history")?; final_sql_builder.order_desc("nft_history.block_timestamp")?; @@ -153,32 +149,21 @@ fn nft_history_table_builder_preimage<'a>( } else if filters.receive && !filters.send { sql_builder.sql_builder().and_where_eq("status", "'Receive'"); } - - if filters.from_date.is_some() && filters.to_date.is_some() { - sql_builder.sql_builder().and_where(format!( - "block_timestamp BETWEEN '{}' and '{}'", - filters.from_date.unwrap(), - filters.to_date.unwrap() - )); - } else if filters.from_date.is_some() { + if let Some(date) = filters.from_date { sql_builder .sql_builder() - .and_where(format!("block_timestamp >= '{}'", filters.from_date.unwrap())); - } else if filters.to_date.is_some() { + .and_where(format!("block_timestamp >= '{}'", date)); + } + if let Some(date) = filters.to_date { sql_builder .sql_builder() - .and_where(format!("block_timestamp <= '{}'", filters.to_date.unwrap())); + .and_where(format!("block_timestamp <= '{}'", date)); } } drop_mutability!(sql_builder); Ok(sql_builder) } -fn nft_table_builder_preimage<'a>(conn: &'a Connection, table_name: &'a str) -> MmResult, SqlError> { - let sql_builder = SqlQuery::select_from(conn, table_name)?; - Ok(sql_builder) -} - fn finalize_nft_list_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { sql_builder.field("nft_list.details_json")?.offset(offset).limit(limit); Ok(()) @@ -217,7 +202,7 @@ fn insert_nft_in_list_sql(chain: &Chain) -> MmResult { validate_table_name(&table_name)?; let sql = format!( - "INSERT OR IGNORE INTO {} ( + "INSERT INTO {} ( token_address, token_id, chain, amount, block_number, contract_type, details_json ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7 @@ -232,7 +217,7 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { validate_table_name(&table_name)?; let sql = format!( - "INSERT OR IGNORE INTO {} ( + "INSERT INTO {} ( transaction_hash, chain, block_number, block_timestamp, contract_type, token_address, token_id, status, amount, collection_name, image, token_name, details_json ) VALUES ( @@ -243,10 +228,14 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { Ok(sql) } -fn insert_chain_into_scanned_blocks_sql() -> MmResult { +fn upsert_last_scanned_block_sql() -> MmResult { let table_name = scanned_nft_blocks_table_name(); validate_table_name(&table_name)?; - Ok(format!("INSERT OR IGNORE INTO {} (chain) VALUES (?1);", table_name)) + let sql = format!( + "INSERT OR REPLACE INTO {} (chain, last_scanned_block) VALUES (?1, ?2);", + table_name + ); + Ok(sql) } fn update_details_json_by_token_add_id_sql(chain: &Chain, table_name_creator: F) -> MmResult @@ -302,13 +291,6 @@ where Ok(sql) } -fn update_last_scanned_block_sql() -> MmResult { - let table_name = scanned_nft_blocks_table_name(); - validate_table_name(&table_name)?; - let sql = format!("UPDATE {} SET last_scanned_block = ?1 WHERE chain = ?2;", table_name); - Ok(sql) -} - fn get_nft_metadata_sql(chain: &Chain) -> MmResult { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; @@ -520,9 +502,8 @@ impl NftListStorageOps for SqliteNftStorage { ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; } - sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, [chain.to_ticker()])?; - let scanned_block_params = [last_scanned_block.to_string(), chain.to_ticker()]; - sql_transaction.execute(&update_last_scanned_block_sql()?, scanned_block_params)?; + let scanned_block_params = [chain.to_ticker(), last_scanned_block.to_string()]; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) @@ -554,8 +535,7 @@ impl NftListStorageOps for SqliteNftStorage { ) -> MmResult { let sql = delete_nft_sql(chain, nft_list_table_name)?; let params = [token_address, token_id.to_string()]; - let chain_param = [chain.to_ticker()]; - let scanned_block_params = [scanned_block.to_string(), chain.to_ticker()]; + let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); @@ -567,8 +547,7 @@ impl NftListStorageOps for SqliteNftStorage { } else { RemoveNftResult::NftDidNotExist }; - sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, chain_param)?; - sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; Ok(remove_nft_result) }) @@ -630,8 +609,7 @@ impl NftListStorageOps for SqliteNftStorage { async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { let sql = update_nft_amount_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); - let chain_param = [chain.to_ticker()]; - let scanned_block_params = [scanned_block.to_string(), chain.to_ticker()]; + let scanned_block_params = [chain.to_ticker(), scanned_block.to_string()]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); @@ -643,8 +621,7 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.token_id.to_string()), ]; sql_transaction.execute(&sql, ¶ms)?; - sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, chain_param)?; - sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) @@ -654,8 +631,7 @@ impl NftListStorageOps for SqliteNftStorage { async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { let sql = update_nft_amount_and_block_number_sql(chain, nft_list_table_name)?; let nft_json = json::to_string(&nft).expect("serialization should not fail"); - let chain_param = [chain.to_ticker()]; - let scanned_block_params = [nft.block_number.to_string(), chain.to_ticker()]; + let scanned_block_params = [chain.to_ticker(), nft.block_number.to_string()]; let selfi = self.clone(); async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); @@ -668,8 +644,7 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.token_id.to_string()), ]; sql_transaction.execute(&sql, ¶ms)?; - sql_transaction.execute(&insert_chain_into_scanned_blocks_sql()?, chain_param)?; - sql_transaction.execute(&update_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, &scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) From f05baafa5125984401c7a5305fe0d29b80b7d586 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 5 Jun 2023 14:55:20 +0700 Subject: [PATCH 059/112] use u64 for block number, from fns for table structs, impl get_nft, get_tx_by_tx_hash --- mm2src/coins/nft.rs | 10 +- mm2src/coins/nft/nft_structs.rs | 2 +- mm2src/coins/nft_storage/mod.rs | 10 +- mm2src/coins/nft_storage/sql_storage.rs | 29 ++-- mm2src/coins/nft_storage/wasm/nft_idb.rs | 2 +- mm2src/coins/nft_storage/wasm/wasm_storage.rs | 134 ++++++++++++++---- 6 files changed, 133 insertions(+), 54 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index b18bc75134..c06b2ec4f5 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -256,7 +256,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult async fn get_moralis_nft_transfers( ctx: &MmArc, chain: &Chain, - from_block: Option, + from_block: Option, url: &Url, ) -> MmResult, GetNftInfoError> { let mut res_list = Vec::new(); @@ -470,7 +470,7 @@ async fn update_nft_list( ctx: MmArc, storage: &T, chain: &Chain, - scan_from_block: u32, + scan_from_block: u64, url: &Url, ) -> MmResult<(), UpdateNftError> { let txs = storage.get_txs_from_block(chain, scan_from_block).await?; @@ -543,7 +543,7 @@ async fn handle_receive_erc721( nft.block_number = tx.block_number; drop_mutability!(nft); storage - .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number as u32) + .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number) .await?; let tx_meta = TxMeta { token_address: nft.token_address, @@ -660,9 +660,7 @@ async fn handle_receive_erc1155( possible_spam: moralis_meta.possible_spam, uri_meta, }; - storage - .add_nfts_to_list(chain, [nft.clone()], tx.block_number as u32) - .await?; + storage.add_nfts_to_list(chain, [nft.clone()], tx.block_number).await?; let tx_meta = TxMeta { token_address: nft.token_address, token_id: nft.token_id, diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 59910bd58f..b2b337ddc5 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -312,7 +312,7 @@ impl fmt::Display for TransferStatus { } } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct NftTransferHistory { pub(crate) chain: Chain, pub(crate) block_number: u64, diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft_storage/mod.rs index 13d7f388af..4fc0b2379f 100644 --- a/mm2src/coins/nft_storage/mod.rs +++ b/mm2src/coins/nft_storage/mod.rs @@ -39,7 +39,7 @@ pub trait NftListStorageOps { page_number: Option, ) -> MmResult; - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u32) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send; @@ -69,11 +69,11 @@ pub trait NftListStorageOps { async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error>; /// `get_last_block_number` function returns the height of last block in NFT LIST table - async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; /// `get_last_scanned_block` function returns the height of last scanned block /// when token was added or removed from MFT LIST table. - async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error>; /// `update_nft_amount` function sets a new amount of a particular token in NFT LIST table async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error>; @@ -105,14 +105,14 @@ pub trait NftTxHistoryStorageOps { I: IntoIterator + Send + 'static, I::IntoIter: Send; - async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error>; /// `get_txs_from_block` function returns transfers sorted by /// block_number in ascending order. It is needed to update the NFT LIST table correctly. async fn get_txs_from_block( &self, chain: &Chain, - from_block: u32, + from_block: u64, ) -> MmResult, Self::Error>; async fn get_txs_by_token_addr_id( diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft_storage/sql_storage.rs index 58ccf4cec8..c97e085a53 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft_storage/sql_storage.rs @@ -344,14 +344,14 @@ where Ok(sql) } -fn block_number_from_row(row: &Row<'_>) -> Result { row.get(0) } +fn block_number_from_row(row: &Row<'_>) -> Result { row.get::<_, i64>(0) } fn nft_amount_from_row(row: &Row<'_>) -> Result { row.get(0) } fn get_txs_from_block_builder<'a>( conn: &'a Connection, chain: &'a Chain, - from_block: u32, + from_block: u64, ) -> MmResult, SqlError> { let table_name = nft_tx_history_table_name(chain); validate_table_name(table_name.as_str())?; @@ -478,7 +478,7 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u32) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -585,17 +585,20 @@ impl NftListStorageOps for SqliteNftStorage { .await } - async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { let sql = select_last_block_number_sql(chain, nft_list_table_name)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); query_single_row(&conn, &sql, NO_PARAMS, block_number_from_row).map_to_mm(SqlError::from) }) - .await + .await? + .map(|b| b.try_into()) + .transpose() + .map_to_mm(|e| SqlError::FromSqlConversionFailure(2, Type::Integer, Box::new(e))) } - async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { let sql = select_last_scanned_block_sql()?; let params = [chain.to_ticker()]; let selfi = self.clone(); @@ -603,7 +606,10 @@ impl NftListStorageOps for SqliteNftStorage { let conn = selfi.0.lock().unwrap(); query_single_row(&conn, &sql, params, block_number_from_row).map_to_mm(SqlError::from) }) - .await + .await? + .map(|b| b.try_into()) + .transpose() + .map_to_mm(|e| SqlError::FromSqlConversionFailure(2, Type::Integer, Box::new(e))) } async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { @@ -753,20 +759,23 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .await } - async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { let sql = select_last_block_number_sql(chain, nft_tx_history_table_name)?; let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); query_single_row(&conn, &sql, NO_PARAMS, block_number_from_row).map_to_mm(SqlError::from) }) - .await + .await? + .map(|b| b.try_into()) + .transpose() + .map_to_mm(|e| SqlError::FromSqlConversionFailure(2, Type::Integer, Box::new(e))) } async fn get_txs_from_block( &self, chain: &Chain, - from_block: u32, + from_block: u64, ) -> MmResult, Self::Error> { let selfi = self.clone(); let chain = *chain; diff --git a/mm2src/coins/nft_storage/wasm/nft_idb.rs b/mm2src/coins/nft_storage/wasm/nft_idb.rs index 89c17ade32..45dece4902 100644 --- a/mm2src/coins/nft_storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft_storage/wasm/nft_idb.rs @@ -29,5 +29,5 @@ impl DbInstance for NftCacheIDB { #[allow(dead_code)] impl NftCacheIDB { - fn get_inner(&self) -> &IndexedDb { &self.inner } + pub(crate) fn get_inner(&self) -> &IndexedDb { &self.inner } } diff --git a/mm2src/coins/nft_storage/wasm/wasm_storage.rs b/mm2src/coins/nft_storage/wasm/wasm_storage.rs index 70dc95c380..b616efc205 100644 --- a/mm2src/coins/nft_storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm/wasm_storage.rs @@ -7,10 +7,11 @@ use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTokenAddrI use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; -use mm2_db::indexed_db::{BeBigUint, DbUpgrader, OnUpgradeResult, SharedDb, TableSignature}; +use mm2_db::indexed_db::{BeBigUint, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; +use mm2_number::num_bigint::ToBigInt; use mm2_number::BigDecimal; use serde_json::{self as json, Value as Json}; use std::num::NonZeroUsize; @@ -52,7 +53,7 @@ impl NftListStorageOps for IndexedDbNftStorage { todo!() } - async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I, _last_scanned_block: u32) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I, _last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, @@ -62,11 +63,22 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn get_nft( &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, + chain: &Chain, + token_address: String, + token_id: BigDecimal, ) -> MmResult, Self::Error> { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)? + .with_value(token_id.to_string())?; + if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { + Ok(Some(nft_details_from_item(item)?)) + } else { + return Ok(None); + } } async fn remove_nft_from_list( @@ -90,9 +102,9 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn refresh_nft_metadata(&self, _chain: &Chain, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } - async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } - async fn get_last_scanned_block(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_last_scanned_block(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } async fn update_nft_amount(&self, _chain: &Chain, _nft: Nft, _scanned_block: u64) -> MmResult<(), Self::Error> { todo!() @@ -130,12 +142,17 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { todo!() } - async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let _table = db_transaction.table::().await?; + todo!() + } async fn get_txs_from_block( &self, _chain: &Chain, - _from_block: u32, + _from_block: u64, ) -> MmResult, Self::Error> { todo!() } @@ -151,10 +168,20 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn get_tx_by_tx_hash( &self, - _chain: &Chain, - _transaction_hash: String, + chain: &Chain, + transaction_hash: String, ) -> MmResult, Self::Error> { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) + .with_value(chain.to_string())? + .with_value(&transaction_hash)?; + if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { + Ok(Some(nft_tx_from_item(item)?)) + } else { + return Ok(None); + } } async fn update_tx_meta_by_hash(&self, _chain: &Chain, _tx: NftTransferHistory) -> MmResult<(), Self::Error> { @@ -171,16 +198,30 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct NftListTable { token_address: String, - token_id: BeBigUint, + token_id: String, chain: String, - amount: BeBigUint, - block_number: u32, - contract_type: ContractType, + amount: String, + block_number: u64, + contract_type: Option, details_json: Json, } impl NftListTable { - pub const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; + const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; + + #[allow(dead_code)] + fn from_nft(nft: &Nft) -> WasmNftCacheResult { + let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; + Ok(NftListTable { + token_address: nft.token_address.clone(), + token_id: nft.token_id.to_string(), + chain: nft.chain.to_string(), + amount: nft.amount.to_string(), + block_number: nft.block_number, + contract_type: nft.contract_type, + details_json, + }) + } } impl TableSignature for NftListTable { @@ -200,30 +241,45 @@ impl TableSignature for NftListTable { } } -#[allow(dead_code)] -fn nft_details_from_item(item: NftListTable) -> WasmNftCacheResult { - json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) -} - #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct NftTxHistoryTable { transaction_hash: String, chain: String, - block_number: u32, + block_number: u64, block_timestamp: String, contract_type: ContractType, token_address: String, - token_id: BeBigUint, + token_id: String, status: TransferStatus, - amount: BeBigUint, - collection_name: String, - image: String, - token_name: String, + amount: String, + collection_name: Option, + image: Option, + token_name: Option, details_json: Json, } impl NftTxHistoryTable { - pub const CHAIN_TX_HASH_INDEX: &str = "chain_tx_hash_index"; + const CHAIN_TX_HASH_INDEX: &str = "chain_tx_hash_index"; + + #[allow(dead_code)] + fn from_tx_history(tx: &NftTransferHistory) -> WasmNftCacheResult { + let details_json = json::to_value(tx).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; + Ok(NftTxHistoryTable { + transaction_hash: tx.transaction_hash.clone(), + chain: tx.chain.to_string(), + block_number: tx.block_number, + block_timestamp: tx.block_timestamp.clone(), + contract_type: tx.contract_type, + token_address: tx.token_address.clone(), + token_id: tx.token_id.to_string(), + status: tx.status, + amount: tx.amount.to_string(), + collection_name: tx.collection_name.clone(), + image: tx.image.clone(), + token_name: tx.token_name.clone(), + details_json, + }) + } } impl TableSignature for NftTxHistoryTable { @@ -247,7 +303,7 @@ fn tx_details_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult WasmNftCacheResult { + json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) +} + +fn nft_tx_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult { + json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) +} + +#[allow(dead_code)] +fn bigdecimal_to_bebiguint(decimal: &BigDecimal) -> Option { + // First convert BigDecimal to BigUint + let biguint = decimal.to_bigint()?.to_biguint()?; + // Then convert BigUint to BeBigUint + Some(BeBigUint::from(biguint)) +} From 480f64898d01870152f9480e4b9c15375cb60458 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 5 Jun 2023 19:05:38 +0700 Subject: [PATCH 060/112] add_nfts_to_list --- mm2src/coins/nft_storage/wasm/wasm_storage.rs | 29 ++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/nft_storage/wasm/wasm_storage.rs b/mm2src/coins/nft_storage/wasm/wasm_storage.rs index b616efc205..871fd6fb07 100644 --- a/mm2src/coins/nft_storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft_storage/wasm/wasm_storage.rs @@ -53,12 +53,27 @@ impl NftListStorageOps for IndexedDbNftStorage { todo!() } - async fn add_nfts_to_list(&self, _chain: &Chain, _nfts: I, _last_scanned_block: u64) -> MmResult<(), Self::Error> + async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let nft_table = db_transaction.table::().await?; + let last_scanned_block_table = db_transaction.table::().await?; + for nft in nfts { + let nft_item = NftListTable::from_nft(&nft)?; + nft_table.add_item(&nft_item).await?; + } + let last_scanned_block = LastScannedBlockTable { + chain: chain.to_string(), + last_scanned_block, + }; + last_scanned_block_table + .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) + .await?; + Ok(()) } async fn get_nft( @@ -178,7 +193,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .with_value(chain.to_string())? .with_value(&transaction_hash)?; if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { - Ok(Some(nft_tx_from_item(item)?)) + Ok(Some(tx_details_from_item(item)?)) } else { return Ok(None); } @@ -209,7 +224,6 @@ pub(crate) struct NftListTable { impl NftListTable { const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; - #[allow(dead_code)] fn from_nft(nft: &Nft) -> WasmNftCacheResult { let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftListTable { @@ -295,11 +309,6 @@ impl TableSignature for NftTxHistoryTable { } } -#[allow(dead_code)] -fn tx_details_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult { - json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) -} - #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct LastScannedBlockTable { chain: String, @@ -322,7 +331,7 @@ fn nft_details_from_item(item: NftListTable) -> WasmNftCacheResult { json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) } -fn nft_tx_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult { +fn tx_details_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult { json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) } From 8291f20a4585550c43722309b4341defc1ea8362 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 5 Jun 2023 20:26:56 +0700 Subject: [PATCH 061/112] make mod storage --- mm2src/coins/lp_coins.rs | 6 ++---- mm2src/coins/nft.rs | 6 ++++-- mm2src/coins/nft/nft_errors.rs | 2 +- mm2src/coins/{nft_storage => nft/storage}/mod.rs | 0 mm2src/coins/{nft_storage => nft/storage}/sql_storage.rs | 4 ++-- mm2src/coins/{nft_storage => nft/storage}/wasm/mod.rs | 2 +- mm2src/coins/{nft_storage => nft/storage}/wasm/nft_idb.rs | 2 +- .../{nft_storage => nft/storage}/wasm/wasm_storage.rs | 8 ++++---- 8 files changed, 15 insertions(+), 15 deletions(-) rename mm2src/coins/{nft_storage => nft/storage}/mod.rs (100%) rename mm2src/coins/{nft_storage => nft/storage}/sql_storage.rs (99%) rename mm2src/coins/{nft_storage => nft/storage}/wasm/mod.rs (98%) rename mm2src/coins/{nft_storage => nft/storage}/wasm/nft_idb.rs (91%) rename mm2src/coins/{nft_storage => nft/storage}/wasm/wasm_storage.rs (95%) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 9d05979104..689ff535b3 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -100,7 +100,7 @@ cfg_wasm32! { use mm2_db::indexed_db::{ConstructibleDb, DbLocked, SharedDb}; use tx_history_storage::wasm::{clear_tx_history, load_tx_history, save_tx_history, TxHistoryDb}; pub type TxHistoryDbLocked<'a> = DbLocked<'a, TxHistoryDb>; - use nft_storage::wasm::nft_idb::NftCacheIDB; + use nft::storage::wasm::nft_idb::NftCacheIDB; } // using custom copy of try_fus as futures crate was renamed to futures01 @@ -286,11 +286,9 @@ use utxo::UtxoActivationParams; use utxo::{BlockchainNetwork, GenerateTxError, UtxoFeeDetails, UtxoTx}; pub mod nft; +use crate::nft::storage::{CreateNftStorageError, NftStorageError}; use nft::nft_errors::GetNftInfoError; -pub(crate) mod nft_storage; -use crate::nft_storage::{CreateNftStorageError, NftStorageError}; - #[cfg(not(target_arch = "wasm32"))] pub mod z_coin; #[cfg(not(target_arch = "wasm32"))] use z_coin::ZCoin; diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index b18bc75134..c066c7cedf 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -4,6 +4,8 @@ use url::Url; pub(crate) mod nft_errors; pub(crate) mod nft_structs; +pub(crate) mod storage; + #[cfg(any(test, target_arch = "wasm32"))] mod nft_tests; use crate::{get_my_address, MyAddressReq, WithdrawError}; @@ -14,7 +16,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, N use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; -use crate::nft_storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; +use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::APPLICATION_JSON; use http::header::ACCEPT; use mm2_err_handle::map_to_mm::MapToMmResult; @@ -152,6 +154,7 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft } pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResult<(), UpdateNftError> { + let storage = NftStorageBuilder::new(&ctx).build()?; let moralis_meta = get_moralis_metadata( format!("{:#02x}", req.token_address), req.token_id.clone(), @@ -159,7 +162,6 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu &req.url, ) .await?; - let storage = NftStorageBuilder::new(&ctx).build()?; let req = NftMetadataReq { token_address: req.token_address, token_id: req.token_id, diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index b3261381ee..485679ccd9 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,5 +1,5 @@ use crate::eth::GetEthAddressError; -use crate::nft_storage::{CreateNftStorageError, NftStorageError}; +use crate::nft::storage::{CreateNftStorageError, NftStorageError}; use crate::GetMyAddressError; use common::HttpStatusCode; use derive_more::Display; diff --git a/mm2src/coins/nft_storage/mod.rs b/mm2src/coins/nft/storage/mod.rs similarity index 100% rename from mm2src/coins/nft_storage/mod.rs rename to mm2src/coins/nft/storage/mod.rs diff --git a/mm2src/coins/nft_storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs similarity index 99% rename from mm2src/coins/nft_storage/sql_storage.rs rename to mm2src/coins/nft/storage/sql_storage.rs index 58ccf4cec8..a0e19b27fc 100644 --- a/mm2src/coins/nft_storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -1,7 +1,7 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTokenAddrId, NftTransferHistory, NftTxHistoryFilters, NftsTransferHistoryList, TxMeta}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps, - RemoveNftResult}; +use crate::nft::storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps, + RemoveNftResult}; use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; diff --git a/mm2src/coins/nft_storage/wasm/mod.rs b/mm2src/coins/nft/storage/wasm/mod.rs similarity index 98% rename from mm2src/coins/nft_storage/wasm/mod.rs rename to mm2src/coins/nft/storage/wasm/mod.rs index 47168452d7..76152223ec 100644 --- a/mm2src/coins/nft_storage/wasm/mod.rs +++ b/mm2src/coins/nft/storage/wasm/mod.rs @@ -1,4 +1,4 @@ -use crate::nft_storage::NftStorageError; +use crate::nft::storage::NftStorageError; use mm2_db::indexed_db::{DbTransactionError, InitDbError}; use mm2_err_handle::prelude::*; diff --git a/mm2src/coins/nft_storage/wasm/nft_idb.rs b/mm2src/coins/nft/storage/wasm/nft_idb.rs similarity index 91% rename from mm2src/coins/nft_storage/wasm/nft_idb.rs rename to mm2src/coins/nft/storage/wasm/nft_idb.rs index 8842cf9b3c..c2e0112289 100644 --- a/mm2src/coins/nft_storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft/storage/wasm/nft_idb.rs @@ -1,4 +1,4 @@ -use crate::nft_storage::wasm::wasm_storage::{NftListTable, NftTxHistoryTable}; +use crate::nft::storage::wasm::wasm_storage::{NftListTable, NftTxHistoryTable}; use async_trait::async_trait; use mm2_db::indexed_db::InitDbResult; use mm2_db::indexed_db::{DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder}; diff --git a/mm2src/coins/nft_storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs similarity index 95% rename from mm2src/coins/nft_storage/wasm/wasm_storage.rs rename to mm2src/coins/nft/storage/wasm/wasm_storage.rs index d60908d647..d9bbb6aa07 100644 --- a/mm2src/coins/nft_storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -1,9 +1,9 @@ use crate::nft::nft_structs::{Chain, ContractType, Nft, NftList, NftTransferHistory, NftsTransferHistoryList, TransferStatus, TxMeta}; -use crate::nft_storage::wasm::nft_idb::{NftCacheIDB, NftCacheIDBLocked}; -use crate::nft_storage::wasm::{WasmNftCacheError, WasmNftCacheResult}; -use crate::nft_storage::{CreateNftStorageError, NftListStorageOps, NftTokenAddrId, NftTxHistoryFilters, - NftTxHistoryStorageOps, RemoveNftResult}; +use crate::nft::storage::wasm::nft_idb::{NftCacheIDB, NftCacheIDBLocked}; +use crate::nft::storage::wasm::{WasmNftCacheError, WasmNftCacheResult}; +use crate::nft::storage::{CreateNftStorageError, NftListStorageOps, NftTokenAddrId, NftTxHistoryFilters, + NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; From a45fd13f81933391c1c6e433e7a42edbe8a83814 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 6 Jun 2023 14:35:46 +0700 Subject: [PATCH 062/112] use prepared stmt in get_nft_list, get_tx_history --- mm2src/coins/nft/storage/sql_storage.rs | 132 +++++++++++++++--------- 1 file changed, 81 insertions(+), 51 deletions(-) diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index a0e19b27fc..6a3d6018cd 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -7,11 +7,11 @@ use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; use db_common::sqlite::rusqlite::types::{FromSqlError, Type}; use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; +use db_common::sqlite::sql_builder::SqlBuilder; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::mm_error::{MmError, MmResult}; -use mm2_err_handle::or_mm_error::OrMmError; use mm2_number::BigDecimal; use serde_json::{self as json}; use std::convert::TryInto; @@ -97,84 +97,101 @@ impl SqliteNftStorage { } } -fn get_nft_list_builder_preimage(conn: &Connection, chains: Vec) -> MmResult { +fn get_nft_list_builder_preimage(chains: Vec) -> MmResult { let union_sql_strings = chains .iter() .map(|chain| { let table_name = nft_list_table_name(chain); validate_table_name(&table_name)?; - let sql_builder = SqlQuery::select_from(conn, table_name.as_str())?; - let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); + let sql_builder = SqlBuilder::select_from(table_name.as_str()); + let sql_string = sql_builder + .sql() + .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))? + .trim_end_matches(';') + .to_string(); Ok(sql_string) }) .collect::, SqlError>>()?; - let union_sql = union_sql_strings.join(" UNION ALL "); - let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_list")?; - final_sql_builder.order_desc("nft_list.block_number")?; + let union_alias_sql = format!("({}) AS nft_list", union_sql_strings.join(" UNION ALL ")); + let mut final_sql_builder = SqlBuilder::select_from(union_alias_sql); + final_sql_builder.order_desc("nft_list.block_number"); drop_mutability!(final_sql_builder); Ok(final_sql_builder) } fn get_nft_tx_builder_preimage( - conn: &Connection, chains: Vec, filters: Option, -) -> MmResult { +) -> MmResult { let union_sql_strings = chains .into_iter() .map(|chain| { let table_name = nft_tx_history_table_name(&chain); validate_table_name(&table_name)?; - let sql_builder = nft_history_table_builder_preimage(conn, table_name.as_str(), filters.clone())?; - let sql_string = sql_builder.sql()?.trim_end_matches(';').to_string(); + let sql_builder = nft_history_table_builder_preimage(table_name.as_str(), filters.clone())?; + let sql_string = sql_builder + .sql() + .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))? + .trim_end_matches(';') + .to_string(); Ok(sql_string) }) .collect::, SqlError>>()?; - let union_sql = union_sql_strings.join(" UNION ALL "); - let mut final_sql_builder = SqlQuery::select_from_union_alias(conn, union_sql.as_str(), "nft_history")?; - final_sql_builder.order_desc("nft_history.block_timestamp")?; + let union_alias_sql = format!("({}) AS nft_history", union_sql_strings.join(" UNION ALL ")); + let mut final_sql_builder = SqlBuilder::select_from(union_alias_sql); + final_sql_builder.order_desc("nft_history.block_timestamp"); drop_mutability!(final_sql_builder); Ok(final_sql_builder) } -fn nft_history_table_builder_preimage<'a>( - conn: &'a Connection, - table_name: &'a str, +fn nft_history_table_builder_preimage( + table_name: &str, filters: Option, -) -> Result, SqlError> { - let mut sql_builder = SqlQuery::select_from(conn, table_name)?; +) -> Result { + let mut sql_builder = SqlBuilder::select_from(table_name); if let Some(filters) = filters { if filters.send && !filters.receive { - sql_builder.sql_builder().and_where_eq("status", "'Send'"); + sql_builder.and_where_eq("status", "'Send'"); } else if filters.receive && !filters.send { - sql_builder.sql_builder().and_where_eq("status", "'Receive'"); + sql_builder.and_where_eq("status", "'Receive'"); } if let Some(date) = filters.from_date { - sql_builder - .sql_builder() - .and_where(format!("block_timestamp >= '{}'", date)); + sql_builder.and_where(format!("block_timestamp >= '{}'", date)); } if let Some(date) = filters.to_date { - sql_builder - .sql_builder() - .and_where(format!("block_timestamp <= '{}'", date)); + sql_builder.and_where(format!("block_timestamp <= '{}'", date)); } } drop_mutability!(sql_builder); Ok(sql_builder) } -fn finalize_nft_list_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { - sql_builder.field("nft_list.details_json")?.offset(offset).limit(limit); - Ok(()) +fn finalize_nft_list_sql_builder( + mut sql_builder: SqlBuilder, + offset: usize, + limit: usize, +) -> MmResult { + let sql = sql_builder + .field("nft_list.details_json") + .offset(offset) + .limit(limit) + .sql() + .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; + Ok(sql) } -fn finalize_nft_history_sql_builder(sql_builder: &mut SqlQuery, offset: usize, limit: usize) -> MmResult<(), SqlError> { - sql_builder - .field("nft_history.details_json")? +fn finalize_nft_history_sql_builder( + mut sql_builder: SqlBuilder, + offset: usize, + limit: usize, +) -> MmResult { + let sql = sql_builder + .field("nft_history.details_json") .offset(offset) - .limit(limit); - Ok(()) + .limit(limit) + .sql() + .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; + Ok(sql) } fn nft_from_row(row: &Row<'_>) -> Result { @@ -450,12 +467,15 @@ impl NftListStorageOps for SqliteNftStorage { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut sql_builder = get_nft_list_builder_preimage(&conn, chains)?; - let mut total_count_builder = sql_builder.clone(); - total_count_builder.count_all()?; - let total: isize = total_count_builder - .query_single_row(|row| row.get(0))? - .or_mm_err(|| SqlError::QueryReturnedNoRows)?; + let sql_builder = get_nft_list_builder_preimage(chains)?; + let total_count_builder_sql = sql_builder + .clone() + .count("*") + .sql() + .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; + let total: isize = conn + .prepare(&total_count_builder_sql)? + .query_row(NO_PARAMS, |row| row.get(0))?; let count_total = total.try_into().expect("count should not be failed"); let (offset, limit) = if max { @@ -466,8 +486,11 @@ impl NftListStorageOps for SqliteNftStorage { None => (0, limit), } }; - finalize_nft_list_sql_builder(&mut sql_builder, offset, limit)?; - let nfts = sql_builder.query(nft_from_row)?; + let sql = finalize_nft_list_sql_builder(sql_builder, offset, limit)?; + let nfts = conn + .prepare(&sql)? + .query_map(NO_PARAMS, nft_from_row)? + .collect::, _>>()?; let result = NftList { nfts, skipped: offset, @@ -690,13 +713,17 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - let mut sql_builder = get_nft_tx_builder_preimage(&conn, chains, filters)?; - let mut total_count_builder = sql_builder.clone(); - total_count_builder.count_all()?; - let total: isize = total_count_builder - .query_single_row(|row| row.get(0))? - .or_mm_err(|| SqlError::QueryReturnedNoRows)?; + let sql_builder = get_nft_tx_builder_preimage(chains, filters)?; + let total_count_builder_sql = sql_builder + .clone() + .count("*") + .sql() + .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; + let total: isize = conn + .prepare(&total_count_builder_sql)? + .query_row(NO_PARAMS, |row| row.get(0))?; let count_total = total.try_into().expect("count should not be failed"); + let (offset, limit) = if max { (0, count_total) } else { @@ -705,8 +732,11 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { None => (0, limit), } }; - finalize_nft_history_sql_builder(&mut sql_builder, offset, limit)?; - let txs = sql_builder.query(tx_history_from_row)?; + let sql = finalize_nft_history_sql_builder(sql_builder, offset, limit)?; + let txs = conn + .prepare(&sql)? + .query_map(NO_PARAMS, tx_history_from_row)? + .collect::, _>>()?; let result = NftsTransferHistoryList { transfer_history: txs, skipped: offset, From 4f16c821b8dd2ac3b2dbae70982d8ea22519304e Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 6 Jun 2023 15:32:56 +0700 Subject: [PATCH 063/112] move impl nft errors from lp_coins into nft mod --- mm2src/coins/lp_coins.rs | 20 -------------------- mm2src/coins/nft/nft_errors.rs | 6 +++++- mm2src/coins/nft/storage/mod.rs | 16 ++++++++++++++++ 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/mm2src/coins/lp_coins.rs b/mm2src/coins/lp_coins.rs index 689ff535b3..7191a172a5 100644 --- a/mm2src/coins/lp_coins.rs +++ b/mm2src/coins/lp_coins.rs @@ -286,7 +286,6 @@ use utxo::UtxoActivationParams; use utxo::{BlockchainNetwork, GenerateTxError, UtxoFeeDetails, UtxoTx}; pub mod nft; -use crate::nft::storage::{CreateNftStorageError, NftStorageError}; use nft::nft_errors::GetNftInfoError; #[cfg(not(target_arch = "wasm32"))] pub mod z_coin; @@ -1932,25 +1931,6 @@ pub enum WithdrawError { DbError(String), } -impl From for WithdrawError { - fn from(e: GetNftInfoError) -> Self { WithdrawError::GetNftInfoError(e) } -} - -impl From for WithdrawError { - fn from(err: T) -> Self { - let msg = format!("{:?}", err); - WithdrawError::DbError(msg) - } -} - -impl From for WithdrawError { - fn from(e: CreateNftStorageError) -> Self { - match e { - CreateNftStorageError::Internal(err) => WithdrawError::InternalError(err), - } - } -} - impl HttpStatusCode for WithdrawError { fn status_code(&self) -> StatusCode { match self { diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 485679ccd9..f4aa3d05c3 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -1,6 +1,6 @@ use crate::eth::GetEthAddressError; use crate::nft::storage::{CreateNftStorageError, NftStorageError}; -use crate::GetMyAddressError; +use crate::{GetMyAddressError, WithdrawError}; use common::HttpStatusCode; use derive_more::Display; use enum_from::EnumFromStringify; @@ -35,6 +35,10 @@ pub enum GetNftInfoError { DbError(String), } +impl From for WithdrawError { + fn from(e: GetNftInfoError) -> Self { WithdrawError::GetNftInfoError(e) } +} + impl From for GetNftInfoError { fn from(e: SlurpError) -> Self { let error_str = e.to_string(); diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 13d7f388af..1f293b691f 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -1,5 +1,6 @@ use crate::nft::nft_structs::{Chain, Nft, NftList, NftTokenAddrId, NftTransferHistory, NftTxHistoryFilters, NftsTransferHistoryList, TxMeta}; +use crate::WithdrawError; use async_trait::async_trait; use derive_more::Display; use mm2_core::mm_ctx::MmArc; @@ -21,6 +22,13 @@ pub enum RemoveNftResult { pub trait NftStorageError: std::fmt::Debug + NotMmError + NotEqual + Send {} +impl From for WithdrawError { + fn from(err: T) -> Self { + let msg = format!("{:?}", err); + WithdrawError::DbError(msg) + } +} + #[async_trait] pub trait NftListStorageOps { type Error: NftStorageError; @@ -140,6 +148,14 @@ pub enum CreateNftStorageError { Internal(String), } +impl From for WithdrawError { + fn from(e: CreateNftStorageError) -> Self { + match e { + CreateNftStorageError::Internal(err) => WithdrawError::InternalError(err), + } + } +} + /// `NftStorageBuilder` is used to create an instance that implements the `NftListStorageOps` /// and `NftTxHistoryStorageOps` traits. pub struct NftStorageBuilder<'a> { From 72a85c1a522bfe1b1c11b524238149f618ff33d7 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 6 Jun 2023 20:17:57 +0700 Subject: [PATCH 064/112] use u64 timestamp for block_timestamp in tx history --- mm2src/coins/nft.rs | 6 ++++-- mm2src/coins/nft/nft_errors.rs | 8 +++++++- mm2src/coins/nft/nft_structs.rs | 6 +++--- mm2src/coins/nft/storage/sql_storage.rs | 8 ++++---- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 2 +- mm2src/common/common.rs | 8 +++++++- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index c066c7cedf..d10596dc51 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -17,7 +17,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, N use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; -use common::APPLICATION_JSON; +use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use http::header::ACCEPT; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_number::BigDecimal; @@ -294,10 +294,12 @@ async fn get_moralis_nft_transfers( for transfer in transfer_list { let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); + let block_timestamp = parse_rfc3339_to_timestamp(&transfer_wrapper.block_timestamp) + .map_to_mm(|e| GetNftInfoError::ParseTimestampError(e.to_string()))?; let transfer_history = NftTransferHistory { chain: *chain, block_number: *transfer_wrapper.block_number, - block_timestamp: transfer_wrapper.block_timestamp, + block_timestamp, block_hash: transfer_wrapper.block_hash, transaction_hash: transfer_wrapper.transaction_hash, transaction_index: transfer_wrapper.transaction_index, diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index f4aa3d05c3..5fd5faf9bd 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -33,6 +33,11 @@ pub enum GetNftInfoError { }, #[display(fmt = "DB error {}", _0)] DbError(String), + #[display( + fmt = "Error parsing datetime to timestamp. Expected format 'YYYY-MM-DDTHH:MM:SS.sssZ', got: {}", + _0 + )] + ParseTimestampError(String), } impl From for WithdrawError { @@ -102,7 +107,8 @@ impl HttpStatusCode for GetNftInfoError { | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) | GetNftInfoError::TokenNotFoundInWallet { .. } - | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::DbError(_) + | GetNftInfoError::ParseTimestampError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 59910bd58f..755f45559d 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -316,7 +316,7 @@ impl fmt::Display for TransferStatus { pub struct NftTransferHistory { pub(crate) chain: Chain, pub(crate) block_number: u64, - pub(crate) block_timestamp: String, + pub(crate) block_timestamp: u64, pub(crate) block_hash: String, /// Transaction hash in hexadecimal format pub(crate) transaction_hash: String, @@ -375,8 +375,8 @@ pub struct NftTxHistoryFilters { pub(crate) receive: bool, #[serde(default)] pub(crate) send: bool, - pub(crate) from_date: Option, - pub(crate) to_date: Option, + pub(crate) from_date: Option, + pub(crate) to_date: Option, } #[derive(Debug, Deserialize)] diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 6a3d6018cd..ec1ddb1182 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -52,7 +52,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { transaction_hash VARCHAR(256) PRIMARY KEY, chain TEXT NOT NULL, block_number INTEGER NOT NULL, - block_timestamp TEXT NOT NULL, + block_timestamp INTEGER NOT NULL, contract_type TEXT NOT NULL, token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, @@ -156,10 +156,10 @@ fn nft_history_table_builder_preimage( sql_builder.and_where_eq("status", "'Receive'"); } if let Some(date) = filters.from_date { - sql_builder.and_where(format!("block_timestamp >= '{}'", date)); + sql_builder.and_where(format!("block_timestamp >= {}", date)); } if let Some(date) = filters.to_date { - sql_builder.and_where(format!("block_timestamp <= '{}'", date)); + sql_builder.and_where(format!("block_timestamp <= {}", date)); } } drop_mutability!(sql_builder); @@ -764,7 +764,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.transaction_hash), Some(tx.chain.to_string()), Some(tx.block_number.to_string()), - Some(tx.block_timestamp), + Some(tx.block_timestamp.to_string()), Some(tx.contract_type.to_string()), Some(tx.token_address), Some(tx.token_id.to_string()), diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index d9bbb6aa07..7834c8ce84 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -205,7 +205,7 @@ pub(crate) struct NftTxHistoryTable { transaction_hash: String, chain: String, block_number: u32, - block_timestamp: String, + block_timestamp: u64, contract_type: ContractType, token_address: String, token_id: BeBigUint, diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index 8670eb3d87..1cc797de95 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -132,7 +132,8 @@ pub mod wio; #[cfg(target_arch = "wasm32")] pub use wasm::*; use backtrace::SymbolName; -use chrono::Utc; +use chrono::format::ParseError; +use chrono::{DateTime, Utc}; pub use futures::compat::Future01CompatExt; use futures01::{future, Future}; use http::header::CONTENT_TYPE; @@ -1014,3 +1015,8 @@ pub fn sha256_digest(path: &PathBuf) -> Result { }; Ok(digest) } + +pub fn parse_rfc3339_to_timestamp(date_str: &str) -> Result { + let date: DateTime = date_str.parse()?; + Ok(date.timestamp() as u64) +} From f7ff6739cbdb536303b981cb57813c0e7fd35eb3 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 8 Jun 2023 18:30:03 +0700 Subject: [PATCH 065/112] NftListStorageOps impl for IDB --- mm2src/coins/nft/storage/wasm/mod.rs | 2 + mm2src/coins/nft/storage/wasm/wasm_storage.rs | 270 +++++++++++++++--- 2 files changed, 238 insertions(+), 34 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/mod.rs b/mm2src/coins/nft/storage/wasm/mod.rs index 76152223ec..c89e27b447 100644 --- a/mm2src/coins/nft/storage/wasm/mod.rs +++ b/mm2src/coins/nft/storage/wasm/mod.rs @@ -18,6 +18,8 @@ pub enum WasmNftCacheError { ErrorClearing(String), NotSupported(String), InternalError(String), + GetLastNftBlockError(String), + GetLastScannedBlockError(String), } impl From for WasmNftCacheError { diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 45fb7afbd8..c997fce897 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -7,13 +7,13 @@ use crate::nft::storage::{CreateNftStorageError, NftListStorageOps, NftTokenAddr use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; -use mm2_db::indexed_db::{BeBigUint, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; +use mm2_db::indexed_db::{DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; -use mm2_number::num_bigint::ToBigInt; use mm2_number::BigDecimal; use serde_json::{self as json, Value as Json}; +use std::cmp::Ordering; use std::num::NonZeroUsize; #[derive(Clone)] @@ -29,10 +29,44 @@ impl IndexedDbNftStorage { }) } - #[allow(dead_code)] async fn lock_db(&self) -> WasmNftCacheResult> { self.db.get_or_initialize().await.mm_err(WasmNftCacheError::from) } + + fn take_nft_according_to_paging_opts( + mut nfts: Vec, + max: bool, + limit: usize, + page_number: Option, + ) -> WasmNftCacheResult { + let total_count = nfts.len(); + nfts.sort_by(compare_nft_block_numbers); + + let (offset, limit) = if max { + (0, total_count) + } else { + match page_number { + Some(page) => ((page.get() - 1) * limit, limit), + None => (0, limit), + } + }; + Ok(NftList { + nfts: nfts.into_iter().skip(offset).take(limit).collect(), + skipped: offset, + total: total_count, + }) + } + + #[allow(dead_code)] + fn take_txs_according_to_filters( + txs: I, + _filters: Option, + ) -> WasmNftCacheResult> + where + I: Iterator, + { + txs.filter(|_tx| todo!()).map(tx_details_from_item).collect() + } } #[async_trait] @@ -45,12 +79,23 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn get_nft_list( &self, - _chains: Vec, - _max: bool, - _limit: usize, - _page_number: Option, + chains: Vec, + max: bool, + limit: usize, + page_number: Option, ) -> MmResult { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let mut nfts = Vec::new(); + for chain in chains { + let items = table.get_items("chain", chain.to_string()).await?; + for (_item_id, item) in items.into_iter() { + let nft_detail = nft_details_from_item(item)?; + nfts.push(nft_detail); + } + } + Self::take_nft_according_to_paging_opts(nfts, max, limit, page_number) } async fn add_nfts_to_list(&self, chain: &Chain, nfts: I, last_scanned_block: u64) -> MmResult<(), Self::Error> @@ -98,35 +143,154 @@ impl NftListStorageOps for IndexedDbNftStorage { async fn remove_nft_from_list( &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, - _scanned_block: u64, + chain: &Chain, + token_address: String, + token_id: BigDecimal, + scanned_block: u64, ) -> MmResult { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let nft_table = db_transaction.table::().await?; + let last_scanned_block_table = db_transaction.table::().await?; + + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)? + .with_value(token_id.to_string())?; + let last_scanned_block = LastScannedBlockTable { + chain: chain.to_string(), + last_scanned_block: scanned_block, + }; + + let nft_removed = nft_table.delete_item_by_unique_multi_index(index_keys).await?.is_some(); + last_scanned_block_table + .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) + .await?; + if nft_removed { + Ok(RemoveNftResult::NftRemoved) + } else { + Ok(RemoveNftResult::NftDidNotExist) + } } async fn get_nft_amount( &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, + chain: &Chain, + token_address: String, + token_id: BigDecimal, ) -> MmResult, Self::Error> { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)? + .with_value(token_id.to_string())?; + if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { + Ok(Some(nft_details_from_item(item)?.amount.to_string())) + } else { + return Ok(None); + } } - async fn refresh_nft_metadata(&self, _chain: &Chain, _nft: Nft) -> MmResult<(), Self::Error> { todo!() } + async fn refresh_nft_metadata(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; - async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&nft.token_address)? + .with_value(nft.token_id.to_string())?; + let nft_item = NftListTable::from_nft(&nft)?; + table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; + Ok(()) + } - async fn get_last_scanned_block(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let maybe_item = table + .cursor_builder() + .only("chain", chain.to_string()) + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .bound("block_number", 0u64, u64::MAX) + .reverse() + .open_cursor(NftListTable::CHAIN_BLOCK_NUMBER_INDEX) + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .next() + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; + Ok(maybe_item.map(|(_, item)| item.block_number)) + } - async fn update_nft_amount(&self, _chain: &Chain, _nft: Nft, _scanned_block: u64) -> MmResult<(), Self::Error> { - todo!() + async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let maybe_item = table + .cursor_builder() + .only("chain", chain.to_string()) + .map_err(|e| WasmNftCacheError::GetLastScannedBlockError(e.to_string()))? + .bound("last_scanned_block", 0u64, u64::MAX) + .reverse() + .open_cursor("chain") + .await + .map_err(|e| WasmNftCacheError::GetLastScannedBlockError(e.to_string()))? + .next() + .await + .map_err(|e| WasmNftCacheError::GetLastScannedBlockError(e.to_string()))?; + Ok(maybe_item.map(|(_, item)| item.last_scanned_block)) } - async fn update_nft_amount_and_block_number(&self, _chain: &Chain, _nft: Nft) -> MmResult<(), Self::Error> { - todo!() + async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let nft_table = db_transaction.table::().await?; + let last_scanned_block_table = db_transaction.table::().await?; + + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&nft.token_address)? + .with_value(nft.token_id.to_string())?; + let nft_item = NftListTable::from_nft(&nft)?; + nft_table + .replace_item_by_unique_multi_index(index_keys, &nft_item) + .await?; + let last_scanned_block = LastScannedBlockTable { + chain: chain.to_string(), + last_scanned_block: scanned_block, + }; + last_scanned_block_table + .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) + .await?; + Ok(()) + } + + async fn update_nft_amount_and_block_number(&self, chain: &Chain, nft: Nft) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let nft_table = db_transaction.table::().await?; + let last_scanned_block_table = db_transaction.table::().await?; + + let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&nft.token_address)? + .with_value(nft.token_id.to_string())?; + let nft_item = NftListTable::from_nft(&nft)?; + nft_table + .replace_item_by_unique_multi_index(index_keys, &nft_item) + .await?; + let last_scanned_block = LastScannedBlockTable { + chain: chain.to_string(), + last_scanned_block: nft.block_number, + }; + last_scanned_block_table + .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) + .await?; + Ok(()) } } @@ -174,11 +338,23 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn get_txs_by_token_addr_id( &self, - _chain: &Chain, - _token_address: String, - _token_id: BigDecimal, + chain: &Chain, + token_address: String, + token_id: BigDecimal, ) -> MmResult, Self::Error> { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) + .with_value(chain.to_string())? + .with_value(&token_address)? + .with_value(token_id.to_string())?; + table + .get_items_by_multi_index(index_keys) + .await? + .into_iter() + .map(|(_item_id, item)| tx_details_from_item(item)) + .collect() } async fn get_tx_by_tx_hash( @@ -222,6 +398,8 @@ pub(crate) struct NftListTable { } impl NftListTable { + const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; + const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; fn from_nft(nft: &Nft) -> WasmNftCacheResult { @@ -244,6 +422,7 @@ impl TableSignature for NftListTable { fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { if let (0, 1) = (old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; + table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; table.create_multi_index( Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, &["chain", "token_address", "token_id"], @@ -273,6 +452,8 @@ pub(crate) struct NftTxHistoryTable { } impl NftTxHistoryTable { + const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; + const CHAIN_TX_HASH_INDEX: &str = "chain_tx_hash_index"; #[allow(dead_code)] @@ -302,6 +483,11 @@ impl TableSignature for NftTxHistoryTable { fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { if let (0, 1) = (old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; + table.create_multi_index( + Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, + &["chain", "token_address", "token_id"], + false, + )?; table.create_multi_index(Self::CHAIN_TX_HASH_INDEX, &["chain", "transaction_hash"], true)?; table.create_index("chain", false)?; } @@ -335,10 +521,26 @@ fn tx_details_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult Option { - // First convert BigDecimal to BigUint - let biguint = decimal.to_bigint()?.to_biguint()?; - // Then convert BigUint to BeBigUint - Some(BeBigUint::from(biguint)) +fn compare_nft_block_numbers(a: &Nft, b: &Nft) -> Ordering { + let a = BlockNumber::new(a.block_number); + let b = BlockNumber::new(b.block_number); + compare_nfts(a, b) +} + +struct BlockNumber { + block_number: u64, +} + +impl BlockNumber { + fn new(block_number: u64) -> BlockNumber { BlockNumber { block_number } } +} + +fn compare_nfts(a: BlockNumber, b: BlockNumber) -> Ordering { + if a.block_number == 0 { + Ordering::Less + } else if b.block_number == 0 { + Ordering::Greater + } else { + b.block_number.cmp(&a.block_number) + } } From 02b9f5f92fd8ce9e5c6f829e1af173114cb48504 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 9 Jun 2023 17:31:49 +0700 Subject: [PATCH 066/112] make contract_type in NftTransferHistory optional --- mm2src/coins/nft.rs | 28 ++++++++++------ mm2src/coins/nft/nft_structs.rs | 6 ++-- mm2src/coins/nft/storage/sql_storage.rs | 44 ++++++++++++------------- 3 files changed, 43 insertions(+), 35 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index d10596dc51..7b8b5c88f6 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -305,7 +305,7 @@ async fn get_moralis_nft_transfers( transaction_index: transfer_wrapper.transaction_index, log_index: transfer_wrapper.log_index, value: transfer_wrapper.value.0, - contract_type: transfer_wrapper.contract_type.0, + contract_type: transfer_wrapper.contract_type.map(|v| v.0), transaction_type: transfer_wrapper.transaction_type, token_address: transfer_wrapper.token_address, token_id: transfer_wrapper.token_id.0, @@ -495,15 +495,23 @@ async fn handle_nft_tx( tx: NftTransferHistory, my_address: &str, ) -> MmResult<(), UpdateNftError> { - match (tx.status, tx.contract_type) { - (TransferStatus::Send, ContractType::Erc721) => handle_send_erc721(storage, chain, tx).await, - (TransferStatus::Receive, ContractType::Erc721) => { - handle_receive_erc721(storage, chain, tx, url, my_address).await - }, - (TransferStatus::Send, ContractType::Erc1155) => handle_send_erc1155(storage, chain, tx).await, - (TransferStatus::Receive, ContractType::Erc1155) => { - handle_receive_erc1155(storage, chain, tx, url, my_address).await - }, + if let Some(contract_type) = tx.contract_type { + match (tx.status, contract_type) { + (TransferStatus::Send, ContractType::Erc721) => handle_send_erc721(storage, chain, tx).await, + (TransferStatus::Receive, ContractType::Erc721) => { + handle_receive_erc721(storage, chain, tx, url, my_address).await + }, + (TransferStatus::Send, ContractType::Erc1155) => handle_send_erc1155(storage, chain, tx).await, + (TransferStatus::Receive, ContractType::Erc1155) => { + handle_receive_erc1155(storage, chain, tx, url, my_address).await + }, + } + } else if tx.status == TransferStatus::Send { + // if contract_type is None, we need to handle this token as erc721, because it is a custom standard like CRYPTO PUNKS + // which has almost the same behaviour as erc721 (1 owner can have 1 nft). + handle_send_erc721(storage, chain, tx).await + } else { + handle_receive_erc721(storage, chain, tx, url, my_address).await } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 755f45559d..88811dd1e5 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -284,7 +284,7 @@ pub(crate) enum ParseTransferStatusError { UnsupportedTransferStatus, } -#[derive(Debug, Deserialize, Clone, Copy, Serialize)] +#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Serialize)] pub(crate) enum TransferStatus { Receive, Send, @@ -323,7 +323,7 @@ pub struct NftTransferHistory { pub(crate) transaction_index: u64, pub(crate) log_index: u64, pub(crate) value: BigDecimal, - pub(crate) contract_type: ContractType, + pub(crate) contract_type: Option, pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: BigDecimal, @@ -349,7 +349,7 @@ pub(crate) struct NftTransferHistoryWrapper { pub(crate) transaction_index: u64, pub(crate) log_index: u64, pub(crate) value: SerdeStringWrap, - pub(crate) contract_type: SerdeStringWrap, + pub(crate) contract_type: Option>, pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: SerdeStringWrap, diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index ec1ddb1182..96d94ae707 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -6,7 +6,7 @@ use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; use db_common::sqlite::rusqlite::types::{FromSqlError, Type}; -use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, NO_PARAMS}; +use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row}; use db_common::sqlite::sql_builder::SqlBuilder; use db_common::sqlite::{query_single_row, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL}; use mm2_core::mm_ctx::MmArc; @@ -53,7 +53,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { chain TEXT NOT NULL, block_number INTEGER NOT NULL, block_timestamp INTEGER NOT NULL, - contract_type TEXT NOT NULL, + contract_type TEXT, token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, status TEXT NOT NULL, @@ -432,8 +432,8 @@ impl NftListStorageOps for SqliteNftStorage { let sql_nft_list = create_nft_list_table_sql(chain)?; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - conn.execute(&sql_nft_list, NO_PARAMS).map(|_| ())?; - conn.execute(&create_scanned_nft_blocks_sql()?, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_nft_list, []).map(|_| ())?; + conn.execute(&create_scanned_nft_blocks_sql()?, []).map(|_| ())?; Ok(()) }) .await @@ -475,7 +475,7 @@ impl NftListStorageOps for SqliteNftStorage { .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; let total: isize = conn .prepare(&total_count_builder_sql)? - .query_row(NO_PARAMS, |row| row.get(0))?; + .query_row([], |row| row.get(0))?; let count_total = total.try_into().expect("count should not be failed"); let (offset, limit) = if max { @@ -489,7 +489,7 @@ impl NftListStorageOps for SqliteNftStorage { let sql = finalize_nft_list_sql_builder(sql_builder, offset, limit)?; let nfts = conn .prepare(&sql)? - .query_map(NO_PARAMS, nft_from_row)? + .query_map([], nft_from_row)? .collect::, _>>()?; let result = NftList { nfts, @@ -523,7 +523,7 @@ impl NftListStorageOps for SqliteNftStorage { nft.contract_type.map(|ct| ct.to_string()), Some(nft_json), ]; - sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, ¶ms)?; + sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, params)?; } let scanned_block_params = [chain.to_ticker(), last_scanned_block.to_string()]; sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; @@ -563,14 +563,14 @@ impl NftListStorageOps for SqliteNftStorage { async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - let rows_num = sql_transaction.execute(&sql, ¶ms)?; + let rows_num = sql_transaction.execute(&sql, params)?; let remove_nft_result = if rows_num > 0 { RemoveNftResult::NftRemoved } else { RemoveNftResult::NftDidNotExist }; - sql_transaction.execute(&upsert_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; sql_transaction.commit()?; Ok(remove_nft_result) }) @@ -601,7 +601,7 @@ impl NftListStorageOps for SqliteNftStorage { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; let params = [nft_json, nft.token_address, nft.token_id.to_string()]; - sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.execute(&sql, params)?; sql_transaction.commit()?; Ok(()) }) @@ -613,7 +613,7 @@ impl NftListStorageOps for SqliteNftStorage { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - query_single_row(&conn, &sql, NO_PARAMS, block_number_from_row).map_to_mm(SqlError::from) + query_single_row(&conn, &sql, [], block_number_from_row).map_to_mm(SqlError::from) }) .await } @@ -643,8 +643,8 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.token_address), Some(nft.token_id.to_string()), ]; - sql_transaction.execute(&sql, ¶ms)?; - sql_transaction.execute(&upsert_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.execute(&sql, params)?; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) @@ -666,8 +666,8 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.token_address), Some(nft.token_id.to_string()), ]; - sql_transaction.execute(&sql, ¶ms)?; - sql_transaction.execute(&upsert_last_scanned_block_sql()?, &scanned_block_params)?; + sql_transaction.execute(&sql, params)?; + sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; sql_transaction.commit()?; Ok(()) }) @@ -684,7 +684,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let sql_tx_history = create_tx_history_table_sql(chain)?; async_blocking(move || { let conn = selfi.0.lock().unwrap(); - conn.execute(&sql_tx_history, NO_PARAMS).map(|_| ())?; + conn.execute(&sql_tx_history, []).map(|_| ())?; Ok(()) }) .await @@ -721,7 +721,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))?; let total: isize = conn .prepare(&total_count_builder_sql)? - .query_row(NO_PARAMS, |row| row.get(0))?; + .query_row([], |row| row.get(0))?; let count_total = total.try_into().expect("count should not be failed"); let (offset, limit) = if max { @@ -735,7 +735,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let sql = finalize_nft_history_sql_builder(sql_builder, offset, limit)?; let txs = conn .prepare(&sql)? - .query_map(NO_PARAMS, tx_history_from_row)? + .query_map([], tx_history_from_row)? .collect::, _>>()?; let result = NftsTransferHistoryList { transfer_history: txs, @@ -765,7 +765,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.chain.to_string()), Some(tx.block_number.to_string()), Some(tx.block_timestamp.to_string()), - Some(tx.contract_type.to_string()), + tx.contract_type.map(|ct| ct.to_string()), Some(tx.token_address), Some(tx.token_id.to_string()), Some(tx.status.to_string()), @@ -775,7 +775,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { tx.token_name, Some(tx_json), ]; - sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, ¶ms)?; + sql_transaction.execute(&insert_tx_in_history_sql(&chain)?, params)?; } sql_transaction.commit()?; Ok(()) @@ -788,7 +788,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let selfi = self.clone(); async_blocking(move || { let conn = selfi.0.lock().unwrap(); - query_single_row(&conn, &sql, NO_PARAMS, block_number_from_row).map_to_mm(SqlError::from) + query_single_row(&conn, &sql, [], block_number_from_row).map_to_mm(SqlError::from) }) .await } @@ -854,7 +854,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - sql_transaction.execute(&sql, ¶ms)?; + sql_transaction.execute(&sql, params)?; sql_transaction.commit()?; Ok(()) }) From 464dbdf17322dfd78ce2873e07bb2da2eb729219 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 12 Jun 2023 15:36:29 +0700 Subject: [PATCH 067/112] polish code in update_nft, move ParseTimestampError from Internal, add None in handle_nft_tx --- mm2src/coins/nft.rs | 136 ++++++++++++++------------------- mm2src/coins/nft/nft_errors.rs | 9 ++- 2 files changed, 64 insertions(+), 81 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 7b8b5c88f6..8ca632ecdf 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -85,70 +85,58 @@ pub async fn update_nft(ctx: MmArc, req: UpdateNftReq) -> MmResult<(), UpdateNft let storage = NftStorageBuilder::new(&ctx).build()?; for chain in req.chains.iter() { let tx_history_initialized = NftTxHistoryStorageOps::is_initialized(&storage, chain).await?; - let list_initialized = NftListStorageOps::is_initialized(&storage, chain).await?; - if !tx_history_initialized { - NftTxHistoryStorageOps::init(&storage, chain).await?; - let nft_transfers = get_moralis_nft_transfers(&ctx, chain, None, &req.url).await?; - storage.add_txs_to_history(chain, nft_transfers).await?; - } else { + let from_block = if tx_history_initialized { let last_tx_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain).await?; - let nft_transfers = get_moralis_nft_transfers(&ctx, chain, last_tx_block.map(|b| b + 1), &req.url).await?; - storage.add_txs_to_history(chain, nft_transfers).await?; - } - - if !list_initialized { - NftListStorageOps::init(&storage, chain).await?; - let nft_list = get_moralis_nft_list(&ctx, chain, &req.url).await?; - let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain) - .await? - .unwrap_or(0); - storage - .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) - .await?; - // this will update only txs related to current nfts in wallet. - update_meta_in_txs(&storage, chain, nft_list).await?; - update_txs_with_empty_meta(&storage, chain, &req.url).await?; + last_tx_block.map(|b| b + 1) } else { - let last_scanned_block = storage.get_last_scanned_block(chain).await?; - let last_nft_block = NftListStorageOps::get_last_block_number(&storage, chain).await?; - - match (last_scanned_block, last_nft_block) { - // if both block numbers exist, last scanned block should be equal - // or higher than last block number from NFT LIST table. - (Some(scanned_block), Some(nft_block)) => { - if scanned_block >= nft_block { - update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; - update_txs_with_empty_meta(&storage, chain, &req.url).await?; - } else { - return MmError::err(UpdateNftError::InvalidBlockOrder { - last_scanned_block: scanned_block.to_string(), - last_nft_block: nft_block.to_string(), - }); - } - }, - // If the last scanned block value is absent, we cannot accurately update the NFT cache. - // This is because a situation may occur where the user doesn't transfer all ERC-1155 tokens, - // resulting in the block number of NFT remaining unchanged. - (None, Some(nft_block)) => { - return MmError::err(UpdateNftError::LastScannedBlockNotFound { - last_nft_block: nft_block.to_string(), - }); - }, - // if there are no rows in NFT LIST table or in both tables there are no rows - // we can try to get all info from moralis. - (Some(_), None) => { - let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; - update_meta_in_txs(&storage, chain, nfts).await?; - update_txs_with_empty_meta(&storage, chain, &req.url).await?; - }, - (None, None) => { - let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; - update_meta_in_txs(&storage, chain, nfts).await?; - update_txs_with_empty_meta(&storage, chain, &req.url).await?; - }, - } + NftTxHistoryStorageOps::init(&storage, chain).await?; + None + }; + let nft_transfers = get_moralis_nft_transfers(&ctx, chain, from_block, &req.url).await?; + storage.add_txs_to_history(chain, nft_transfers).await?; + + let nft_block = match NftListStorageOps::get_last_block_number(&storage, chain).await { + Ok(Some(block)) => block, + Ok(None) => { + // if there are no rows in NFT LIST table we can try to get all info from moralis. + let nfts = cache_nfts_from_moralis(&ctx, &storage, chain, &req.url).await?; + update_meta_in_txs(&storage, chain, nfts).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; + continue; + }, + Err(_) => { + // if there is an error, then NFT LIST table doesnt exist, so we need to cache from mroalis. + NftListStorageOps::init(&storage, chain).await?; + let nft_list = get_moralis_nft_list(&ctx, chain, &req.url).await?; + let last_scanned_block = NftTxHistoryStorageOps::get_last_block_number(&storage, chain) + .await? + .unwrap_or(0); + storage + .add_nfts_to_list(chain, nft_list.clone(), last_scanned_block) + .await?; + update_meta_in_txs(&storage, chain, nft_list).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; + continue; + }, + }; + let scanned_block = + storage + .get_last_scanned_block(chain) + .await? + .ok_or_else(|| UpdateNftError::LastScannedBlockNotFound { + last_nft_block: nft_block.to_string(), + })?; + // if both block numbers exist, last scanned block should be equal + // or higher than last block number from NFT LIST table. + if scanned_block < nft_block { + return MmError::err(UpdateNftError::InvalidBlockOrder { + last_scanned_block: scanned_block.to_string(), + last_nft_block: nft_block.to_string(), + }); } + update_nft_list(ctx.clone(), &storage, chain, scanned_block + 1, &req.url).await?; + update_txs_with_empty_meta(&storage, chain, &req.url).await?; } Ok(()) } @@ -495,23 +483,15 @@ async fn handle_nft_tx( tx: NftTransferHistory, my_address: &str, ) -> MmResult<(), UpdateNftError> { - if let Some(contract_type) = tx.contract_type { - match (tx.status, contract_type) { - (TransferStatus::Send, ContractType::Erc721) => handle_send_erc721(storage, chain, tx).await, - (TransferStatus::Receive, ContractType::Erc721) => { - handle_receive_erc721(storage, chain, tx, url, my_address).await - }, - (TransferStatus::Send, ContractType::Erc1155) => handle_send_erc1155(storage, chain, tx).await, - (TransferStatus::Receive, ContractType::Erc1155) => { - handle_receive_erc1155(storage, chain, tx, url, my_address).await - }, - } - } else if tx.status == TransferStatus::Send { - // if contract_type is None, we need to handle this token as erc721, because it is a custom standard like CRYPTO PUNKS - // which has almost the same behaviour as erc721 (1 owner can have 1 nft). - handle_send_erc721(storage, chain, tx).await - } else { - handle_receive_erc721(storage, chain, tx, url, my_address).await + match (tx.status, tx.contract_type) { + (TransferStatus::Send, None | Some(ContractType::Erc721)) => handle_send_erc721(storage, chain, tx).await, + (TransferStatus::Receive, None | Some(ContractType::Erc721)) => { + handle_receive_erc721(storage, chain, tx, url, my_address).await + }, + (TransferStatus::Send, Some(ContractType::Erc1155)) => handle_send_erc1155(storage, chain, tx).await, + (TransferStatus::Receive, Some(ContractType::Erc1155)) => { + handle_receive_erc1155(storage, chain, tx, url, my_address).await + }, } } @@ -725,6 +705,7 @@ async fn cache_nfts_from_moralis( Ok(nft_list) } +/// `update_meta_in_txs` function updates only txs related to current nfts in wallet. async fn update_meta_in_txs(storage: &T, chain: &Chain, nfts: Vec) -> MmResult<(), UpdateNftError> where T: NftListStorageOps + NftTxHistoryStorageOps, @@ -742,6 +723,7 @@ where Ok(()) } +/// `update_txs_with_empty_meta` function updates empty metadata in transfers. async fn update_txs_with_empty_meta(storage: &T, chain: &Chain, url: &Url) -> MmResult<(), UpdateNftError> where T: NftListStorageOps + NftTxHistoryStorageOps, diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 5fd5faf9bd..2d89aba9e8 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -102,13 +102,14 @@ impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { GetNftInfoError::InvalidRequest(_) => StatusCode::BAD_REQUEST, - GetNftInfoError::InvalidResponse(_) => StatusCode::FAILED_DEPENDENCY, + GetNftInfoError::InvalidResponse(_) | GetNftInfoError::ParseTimestampError(_) => { + StatusCode::FAILED_DEPENDENCY + }, GetNftInfoError::Transport(_) | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) | GetNftInfoError::TokenNotFoundInWallet { .. } - | GetNftInfoError::DbError(_) - | GetNftInfoError::ParseTimestampError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -141,7 +142,7 @@ pub enum UpdateNftError { amount_history: String, }, #[display( - fmt = "Last scanned nft block {} should be >= last block number in nft table {}", + fmt = "Last scanned nft block {} should be >= last block number {} in nft table", last_scanned_block, last_nft_block )] From 8be6ab0d4ad6c8e76462105a1ea2017d6749a251 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 12 Jun 2023 21:05:12 +0700 Subject: [PATCH 068/112] make contract_type mandatory --- mm2src/coins/nft.rs | 14 +++++++------- mm2src/coins/nft/nft_structs.rs | 8 ++++---- mm2src/coins/nft/storage/sql_storage.rs | 8 ++++---- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 8ca632ecdf..a879ac2751 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -215,7 +215,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult token_hash: nft_wrapper.token_hash, block_number_minted: *nft_wrapper.block_number_minted, block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.map(|v| v.0), + contract_type: nft_wrapper.contract_type.0, collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, token_uri: nft_wrapper.token_uri, @@ -293,7 +293,7 @@ async fn get_moralis_nft_transfers( transaction_index: transfer_wrapper.transaction_index, log_index: transfer_wrapper.log_index, value: transfer_wrapper.value.0, - contract_type: transfer_wrapper.contract_type.map(|v| v.0), + contract_type: transfer_wrapper.contract_type.0, transaction_type: transfer_wrapper.transaction_type, token_address: transfer_wrapper.token_address, token_id: transfer_wrapper.token_id.0, @@ -358,7 +358,7 @@ async fn get_moralis_metadata( token_hash: nft_wrapper.token_hash, block_number_minted: *nft_wrapper.block_number_minted, block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.map(|v| v.0), + contract_type: nft_wrapper.contract_type.0, collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, token_uri: nft_wrapper.token_uri, @@ -484,12 +484,12 @@ async fn handle_nft_tx( my_address: &str, ) -> MmResult<(), UpdateNftError> { match (tx.status, tx.contract_type) { - (TransferStatus::Send, None | Some(ContractType::Erc721)) => handle_send_erc721(storage, chain, tx).await, - (TransferStatus::Receive, None | Some(ContractType::Erc721)) => { + (TransferStatus::Send, ContractType::Erc721) => handle_send_erc721(storage, chain, tx).await, + (TransferStatus::Receive, ContractType::Erc721) => { handle_receive_erc721(storage, chain, tx, url, my_address).await }, - (TransferStatus::Send, Some(ContractType::Erc1155)) => handle_send_erc1155(storage, chain, tx).await, - (TransferStatus::Receive, Some(ContractType::Erc1155)) => { + (TransferStatus::Send, ContractType::Erc1155) => handle_send_erc1155(storage, chain, tx).await, + (TransferStatus::Receive, ContractType::Erc1155) => { handle_receive_erc1155(storage, chain, tx, url, my_address).await }, } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 88811dd1e5..3700e5086e 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -148,7 +148,7 @@ pub struct Nft { pub(crate) token_hash: String, pub(crate) block_number_minted: u64, pub(crate) block_number: u64, - pub(crate) contract_type: Option, + pub(crate) contract_type: ContractType, pub(crate) collection_name: Option, pub(crate) symbol: Option, pub(crate) token_uri: Option, @@ -171,7 +171,7 @@ pub(crate) struct NftWrapper { pub(crate) token_hash: String, pub(crate) block_number_minted: SerdeStringWrap, pub(crate) block_number: SerdeStringWrap, - pub(crate) contract_type: Option>, + pub(crate) contract_type: SerdeStringWrap, pub(crate) name: Option, pub(crate) symbol: Option, pub(crate) token_uri: Option, @@ -323,7 +323,7 @@ pub struct NftTransferHistory { pub(crate) transaction_index: u64, pub(crate) log_index: u64, pub(crate) value: BigDecimal, - pub(crate) contract_type: Option, + pub(crate) contract_type: ContractType, pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: BigDecimal, @@ -349,7 +349,7 @@ pub(crate) struct NftTransferHistoryWrapper { pub(crate) transaction_index: u64, pub(crate) log_index: u64, pub(crate) value: SerdeStringWrap, - pub(crate) contract_type: Option>, + pub(crate) contract_type: SerdeStringWrap, pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: SerdeStringWrap, diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 96d94ae707..5e9cee8cb7 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -35,7 +35,7 @@ fn create_nft_list_table_sql(chain: &Chain) -> MmResult { chain TEXT NOT NULL, amount VARCHAR(256) NOT NULL, block_number INTEGER NOT NULL, - contract_type TEXT, + contract_type TEXT NOT NULL, details_json TEXT, PRIMARY KEY (token_address, token_id) );", @@ -53,7 +53,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { chain TEXT NOT NULL, block_number INTEGER NOT NULL, block_timestamp INTEGER NOT NULL, - contract_type TEXT, + contract_type TEXT NOT NULL, token_address VARCHAR(256) NOT NULL, token_id VARCHAR(256) NOT NULL, status TEXT NOT NULL, @@ -520,7 +520,7 @@ impl NftListStorageOps for SqliteNftStorage { Some(nft.chain.to_string()), Some(nft.amount.to_string()), Some(nft.block_number.to_string()), - nft.contract_type.map(|ct| ct.to_string()), + Some(nft.contract_type.to_string()), Some(nft_json), ]; sql_transaction.execute(&insert_nft_in_list_sql(&chain)?, params)?; @@ -765,7 +765,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.chain.to_string()), Some(tx.block_number.to_string()), Some(tx.block_timestamp.to_string()), - tx.contract_type.map(|ct| ct.to_string()), + Some(tx.contract_type.to_string()), Some(tx.token_address), Some(tx.token_id.to_string()), Some(tx.status.to_string()), From 8e3be13eab54e460ca7975cb0caef15eb2ccfb9a Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 14 Jun 2023 12:56:10 +0700 Subject: [PATCH 069/112] contract_type is some check --- mm2src/coins/nft.rs | 113 +++++++++++++++++--------------- mm2src/coins/nft/nft_errors.rs | 8 ++- mm2src/coins/nft/nft_structs.rs | 4 +- 3 files changed, 67 insertions(+), 58 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index a879ac2751..1c631db638 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -205,29 +205,31 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult if let Some(nfts_list) = response["result"].as_array() { for nft_json in nfts_list { let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; - let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; - let nft = Nft { - chain: *chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, - block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.0, - collection_name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, - uri_meta, - }; - // collect NFTs from the page - res_list.push(nft); + if nft_wrapper.contract_type.is_some() { + let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; + let nft = Nft { + chain: *chain, + token_address: nft_wrapper.token_address, + token_id: nft_wrapper.token_id.0, + amount: nft_wrapper.amount.0, + owner_of: nft_wrapper.owner_of, + token_hash: nft_wrapper.token_hash, + block_number_minted: *nft_wrapper.block_number_minted, + block_number: *nft_wrapper.block_number, + contract_type: nft_wrapper.contract_type.unwrap().0, + collection_name: nft_wrapper.name, + symbol: nft_wrapper.symbol, + token_uri: nft_wrapper.token_uri, + metadata: nft_wrapper.metadata, + last_token_uri_sync: nft_wrapper.last_token_uri_sync, + last_metadata_sync: nft_wrapper.last_metadata_sync, + minter_address: nft_wrapper.minter_address, + possible_spam: nft_wrapper.possible_spam, + uri_meta, + }; + // collect NFTs from the page + res_list.push(nft); + } } // if cursor is not null, there are other NFTs on next page, // and we need to send new request with cursor to get info from the next page. @@ -281,35 +283,37 @@ async fn get_moralis_nft_transfers( if let Some(transfer_list) = response["result"].as_array() { for transfer in transfer_list { let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; - let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); - let block_timestamp = parse_rfc3339_to_timestamp(&transfer_wrapper.block_timestamp) - .map_to_mm(|e| GetNftInfoError::ParseTimestampError(e.to_string()))?; - let transfer_history = NftTransferHistory { - chain: *chain, - block_number: *transfer_wrapper.block_number, - block_timestamp, - block_hash: transfer_wrapper.block_hash, - transaction_hash: transfer_wrapper.transaction_hash, - transaction_index: transfer_wrapper.transaction_index, - log_index: transfer_wrapper.log_index, - value: transfer_wrapper.value.0, - contract_type: transfer_wrapper.contract_type.0, - transaction_type: transfer_wrapper.transaction_type, - token_address: transfer_wrapper.token_address, - token_id: transfer_wrapper.token_id.0, - collection_name: None, - image: None, - token_name: None, - from_address: transfer_wrapper.from_address, - to_address: transfer_wrapper.to_address, - status, - amount: transfer_wrapper.amount.0, - verified: transfer_wrapper.verified, - operator: transfer_wrapper.operator, - possible_spam: transfer_wrapper.possible_spam, - }; - // collect NFTs transfers from the page - res_list.push(transfer_history); + if transfer_wrapper.contract_type.is_some() { + let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); + let block_timestamp = parse_rfc3339_to_timestamp(&transfer_wrapper.block_timestamp) + .map_to_mm(|e| GetNftInfoError::ParseTimestampError(e.to_string()))?; + let transfer_history = NftTransferHistory { + chain: *chain, + block_number: *transfer_wrapper.block_number, + block_timestamp, + block_hash: transfer_wrapper.block_hash, + transaction_hash: transfer_wrapper.transaction_hash, + transaction_index: transfer_wrapper.transaction_index, + log_index: transfer_wrapper.log_index, + value: transfer_wrapper.value.0, + contract_type: transfer_wrapper.contract_type.unwrap().0, + transaction_type: transfer_wrapper.transaction_type, + token_address: transfer_wrapper.token_address, + token_id: transfer_wrapper.token_id.0, + collection_name: None, + image: None, + token_name: None, + from_address: transfer_wrapper.from_address, + to_address: transfer_wrapper.to_address, + status, + amount: transfer_wrapper.amount.0, + verified: transfer_wrapper.verified, + operator: transfer_wrapper.operator, + possible_spam: transfer_wrapper.possible_spam, + }; + // collect NFTs transfers from the page + res_list.push(transfer_history); + } } // if the cursor is not null, there are other NFTs transfers on next page, // and we need to send new request with cursor to get info from the next page. @@ -348,6 +352,9 @@ async fn get_moralis_metadata( let response = send_request_to_uri(uri.as_str()).await?; let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string())?; + if nft_wrapper.contract_type.is_none() { + return MmError::err(GetNftInfoError::ContractTypeIsNull); + } let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; let nft_metadata = Nft { chain: *chain, @@ -358,7 +365,7 @@ async fn get_moralis_metadata( token_hash: nft_wrapper.token_hash, block_number_minted: *nft_wrapper.block_number_minted, block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.0, + contract_type: nft_wrapper.contract_type.unwrap().0, collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, token_uri: nft_wrapper.token_uri, diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 2d89aba9e8..a5defb9beb 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -38,6 +38,8 @@ pub enum GetNftInfoError { _0 )] ParseTimestampError(String), + #[display(fmt = "The contract type is required and should not be null.")] + ContractTypeIsNull, } impl From for WithdrawError { @@ -102,9 +104,9 @@ impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { GetNftInfoError::InvalidRequest(_) => StatusCode::BAD_REQUEST, - GetNftInfoError::InvalidResponse(_) | GetNftInfoError::ParseTimestampError(_) => { - StatusCode::FAILED_DEPENDENCY - }, + GetNftInfoError::InvalidResponse(_) + | GetNftInfoError::ParseTimestampError(_) + | GetNftInfoError::ContractTypeIsNull => StatusCode::FAILED_DEPENDENCY, GetNftInfoError::Transport(_) | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 3700e5086e..f48dc4d266 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -171,7 +171,7 @@ pub(crate) struct NftWrapper { pub(crate) token_hash: String, pub(crate) block_number_minted: SerdeStringWrap, pub(crate) block_number: SerdeStringWrap, - pub(crate) contract_type: SerdeStringWrap, + pub(crate) contract_type: Option>, pub(crate) name: Option, pub(crate) symbol: Option, pub(crate) token_uri: Option, @@ -349,7 +349,7 @@ pub(crate) struct NftTransferHistoryWrapper { pub(crate) transaction_index: u64, pub(crate) log_index: u64, pub(crate) value: SerdeStringWrap, - pub(crate) contract_type: SerdeStringWrap, + pub(crate) contract_type: Option>, pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: SerdeStringWrap, From b3f7a1ec6f913d0331d1dac4ac765bba6ee58510 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 12 Jun 2023 20:22:39 +0700 Subject: [PATCH 070/112] NftTxHistoryStorageOps for IDB --- mm2src/coins/nft/nft_structs.rs | 10 +- mm2src/coins/nft/storage/mod.rs | 10 + mm2src/coins/nft/storage/sql_storage.rs | 24 +- mm2src/coins/nft/storage/wasm/mod.rs | 2 + mm2src/coins/nft/storage/wasm/wasm_storage.rs | 227 +++++++++++++----- 5 files changed, 188 insertions(+), 85 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 0bfe1a174c..d26966e2e1 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -284,7 +284,7 @@ pub(crate) enum ParseTransferStatusError { UnsupportedTransferStatus, } -#[derive(Debug, Deserialize, Clone, Copy, Serialize)] +#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Serialize)] pub(crate) enum TransferStatus { Receive, Send, @@ -368,11 +368,10 @@ pub struct NftsTransferHistoryList { pub(crate) total: usize, } -#[allow(dead_code)] -#[derive(Clone, Debug, Deserialize)] +#[derive(Copy, Clone, Debug, Deserialize)] pub struct NftTxHistoryFilters { #[serde(default)] - pub(crate) receive: bool, + pub receive: bool, #[serde(default)] pub(crate) send: bool, pub(crate) from_date: Option, @@ -385,13 +384,12 @@ pub struct UpdateNftReq { pub(crate) url: Url, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Eq, Hash, PartialEq)] pub struct NftTokenAddrId { pub(crate) token_address: String, pub(crate) token_id: BigDecimal, } -#[allow(dead_code)] #[derive(Debug)] pub struct TxMeta { pub(crate) token_address: String, diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 9ba4761669..b0cdc8a9b3 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -174,3 +174,13 @@ impl<'a> NftStorageBuilder<'a> { sql_storage::SqliteNftStorage::new(self.ctx) } } + +fn get_offset_limit(max: bool, limit: usize, page_number: Option, total_count: usize) -> (usize, usize) { + if max { + return (0, total_count); + } + match page_number { + Some(page) => ((page.get() - 1) * limit, limit), + None => (0, limit), + } +} diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 91d0fcf0b0..5bc4943934 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -1,7 +1,7 @@ use crate::nft::nft_structs::{Chain, ConvertChain, Nft, NftList, NftTokenAddrId, NftTransferHistory, NftTxHistoryFilters, NftsTransferHistoryList, TxMeta}; -use crate::nft::storage::{CreateNftStorageError, NftListStorageOps, NftStorageError, NftTxHistoryStorageOps, - RemoveNftResult}; +use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorageOps, NftStorageError, + NftTxHistoryStorageOps, RemoveNftResult}; use async_trait::async_trait; use common::async_blocking; use db_common::sql_build::{SqlCondition, SqlQuery}; @@ -128,7 +128,7 @@ fn get_nft_tx_builder_preimage( .map(|chain| { let table_name = nft_tx_history_table_name(&chain); validate_table_name(&table_name)?; - let sql_builder = nft_history_table_builder_preimage(table_name.as_str(), filters.clone())?; + let sql_builder = nft_history_table_builder_preimage(table_name.as_str(), filters)?; let sql_string = sql_builder .sql() .map_err(|e| SqlError::ToSqlConversionFailure(e.into()))? @@ -478,14 +478,7 @@ impl NftListStorageOps for SqliteNftStorage { .query_row(NO_PARAMS, |row| row.get(0))?; let count_total = total.try_into().expect("count should not be failed"); - let (offset, limit) = if max { - (0, count_total) - } else { - match page_number { - Some(page) => ((page.get() - 1) * limit, limit), - None => (0, limit), - } - }; + let (offset, limit) = get_offset_limit(max, limit, page_number, count_total); let sql = finalize_nft_list_sql_builder(sql_builder, offset, limit)?; let nfts = conn .prepare(&sql)? @@ -730,14 +723,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .query_row(NO_PARAMS, |row| row.get(0))?; let count_total = total.try_into().expect("count should not be failed"); - let (offset, limit) = if max { - (0, count_total) - } else { - match page_number { - Some(page) => ((page.get() - 1) * limit, limit), - None => (0, limit), - } - }; + let (offset, limit) = get_offset_limit(max, limit, page_number, count_total); let sql = finalize_nft_history_sql_builder(sql_builder, offset, limit)?; let txs = conn .prepare(&sql)? diff --git a/mm2src/coins/nft/storage/wasm/mod.rs b/mm2src/coins/nft/storage/wasm/mod.rs index c89e27b447..dd4e58568b 100644 --- a/mm2src/coins/nft/storage/wasm/mod.rs +++ b/mm2src/coins/nft/storage/wasm/mod.rs @@ -1,6 +1,7 @@ use crate::nft::storage::NftStorageError; use mm2_db::indexed_db::{DbTransactionError, InitDbError}; use mm2_err_handle::prelude::*; +use mm2_number::bigdecimal::ParseBigDecimalError; pub mod nft_idb; pub mod wasm_storage; @@ -20,6 +21,7 @@ pub enum WasmNftCacheError { InternalError(String), GetLastNftBlockError(String), GetLastScannedBlockError(String), + ParseBigDecimalError(ParseBigDecimalError), } impl From for WasmNftCacheError { diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index c997fce897..9f6a5c6639 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -2,8 +2,8 @@ use crate::nft::nft_structs::{Chain, ContractType, Nft, NftList, NftTransferHist TransferStatus, TxMeta}; use crate::nft::storage::wasm::nft_idb::{NftCacheIDB, NftCacheIDBLocked}; use crate::nft::storage::wasm::{WasmNftCacheError, WasmNftCacheResult}; -use crate::nft::storage::{CreateNftStorageError, NftListStorageOps, NftTokenAddrId, NftTxHistoryFilters, - NftTxHistoryStorageOps, RemoveNftResult}; +use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorageOps, NftTokenAddrId, + NftTxHistoryFilters, NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; @@ -13,8 +13,9 @@ use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; use mm2_number::BigDecimal; use serde_json::{self as json, Value as Json}; -use std::cmp::Ordering; +use std::collections::HashSet; use std::num::NonZeroUsize; +use std::str::FromStr; #[derive(Clone)] pub struct IndexedDbNftStorage { @@ -40,16 +41,8 @@ impl IndexedDbNftStorage { page_number: Option, ) -> WasmNftCacheResult { let total_count = nfts.len(); - nfts.sort_by(compare_nft_block_numbers); - - let (offset, limit) = if max { - (0, total_count) - } else { - match page_number { - Some(page) => ((page.get() - 1) * limit, limit), - None => (0, limit), - } - }; + nfts.sort_by(|a, b| b.block_number.cmp(&a.block_number)); + let (offset, limit) = get_offset_limit(max, limit, page_number, total_count); Ok(NftList { nfts: nfts.into_iter().skip(offset).take(limit).collect(), skipped: offset, @@ -57,15 +50,47 @@ impl IndexedDbNftStorage { }) } - #[allow(dead_code)] + fn take_txs_according_to_paging_opts( + mut txs: Vec, + max: bool, + limit: usize, + page_number: Option, + ) -> WasmNftCacheResult { + let total_count = txs.len(); + txs.sort_by(|a, b| b.block_timestamp.cmp(&a.block_timestamp)); + let (offset, limit) = get_offset_limit(max, limit, page_number, total_count); + Ok(NftsTransferHistoryList { + transfer_history: txs.into_iter().skip(offset).take(limit).collect(), + skipped: offset, + total: total_count, + }) + } + fn take_txs_according_to_filters( txs: I, - _filters: Option, + filters: Option, ) -> WasmNftCacheResult> where I: Iterator, { - txs.filter(|_tx| todo!()).map(tx_details_from_item).collect() + let mut filtered_txs = Vec::new(); + + for tx_table in txs { + let tx = tx_details_from_item(tx_table)?; + if let Some(filters) = &filters { + let status_matches = (filters.receive && tx.status == TransferStatus::Receive) + || (filters.send && tx.status == TransferStatus::Send); + let date_matches = filters.from_date.map_or(true, |from| tx.block_timestamp >= from) + && filters.to_date.map_or(true, |to| tx.block_timestamp <= to); + if status_matches && date_matches { + filtered_txs.push(tx); + } + } else { + filtered_txs.push(tx); + } + } + drop_mutability!(filtered_txs); + Ok(filtered_txs) } } @@ -304,36 +329,90 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { async fn get_tx_history( &self, - _chains: Vec, - _max: bool, - _limit: usize, - _page_number: Option, - _filters: Option, + chains: Vec, + max: bool, + limit: usize, + page_number: Option, + filters: Option, ) -> MmResult { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let mut txs = Vec::new(); + for chain in chains { + let tx_tables = table + .get_items("chain", chain.to_string()) + .await? + .into_iter() + .map(|(_item_id, tx)| tx); + let filtered = Self::take_txs_according_to_filters(tx_tables, filters)?; + txs.extend(filtered); + } + Self::take_txs_according_to_paging_opts(txs, max, limit, page_number) } - async fn add_txs_to_history(&self, _chain: &Chain, _txs: I) -> MmResult<(), Self::Error> + async fn add_txs_to_history(&self, _chain: &Chain, txs: I) -> MmResult<(), Self::Error> where I: IntoIterator + Send + 'static, I::IntoIter: Send, { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + for tx in txs { + let tx_item = NftTxHistoryTable::from_tx_history(&tx)?; + table.add_item(&tx_item).await?; + } + Ok(()) } - async fn get_last_block_number(&self, _chain: &Chain) -> MmResult, Self::Error> { + async fn get_last_block_number(&self, chain: &Chain) -> MmResult, Self::Error> { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; - let _table = db_transaction.table::().await?; - todo!() + let table = db_transaction.table::().await?; + let maybe_item = table + .cursor_builder() + .only("chain", chain.to_string()) + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .bound("block_number", 0u64, u64::MAX) + .reverse() + .open_cursor("chain") + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .next() + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; + Ok(maybe_item.map(|(_, item)| item.block_number)) } async fn get_txs_from_block( &self, - _chain: &Chain, - _from_block: u64, + chain: &Chain, + from_block: u64, ) -> MmResult, Self::Error> { - todo!() + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let mut cursor = table + .cursor_builder() + .only("chain", chain.to_string()) + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .bound("block_number", from_block, u64::MAX) + .open_cursor("chain") + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; + + let mut res = Vec::new(); + while let Some((_item_id, item)) = cursor + .next() + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + { + let tx = tx_details_from_item(item)?; + res.push(tx); + } + drop_mutability!(res); + Ok(res) } async fn get_txs_by_token_addr_id( @@ -348,6 +427,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? + .with_value(&token_address)? .with_value(token_id.to_string())?; table .get_items_by_multi_index(index_keys) @@ -375,15 +455,67 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { } } - async fn update_tx_meta_by_hash(&self, _chain: &Chain, _tx: NftTransferHistory) -> MmResult<(), Self::Error> { - todo!() + async fn update_tx_meta_by_hash(&self, chain: &Chain, tx: NftTransferHistory) -> MmResult<(), Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) + .with_value(chain.to_string())? + .with_value(&tx.transaction_hash)?; + let item = NftTxHistoryTable::from_tx_history(&tx)?; + table.replace_item_by_unique_multi_index(index_keys, &item).await?; + Ok(()) } - async fn update_txs_meta_by_token_addr_id(&self, _chain: &Chain, _tx_meta: TxMeta) -> MmResult<(), Self::Error> { - todo!() + async fn update_txs_meta_by_token_addr_id(&self, chain: &Chain, tx_meta: TxMeta) -> MmResult<(), Self::Error> { + let txs: Vec = self + .get_txs_by_token_addr_id(chain, tx_meta.token_address, tx_meta.token_id) + .await?; + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + for mut tx in txs { + tx.collection_name = tx_meta.collection_name.clone(); + tx.image = tx_meta.image.clone(); + tx.token_name = tx_meta.token_name.clone(); + drop_mutability!(tx); + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) + .with_value(chain.to_string())? + .with_value(&tx.transaction_hash)?; + let item = NftTxHistoryTable::from_tx_history(&tx)?; + table.replace_item_by_unique_multi_index(index_keys, &item).await?; + } + Ok(()) } - async fn get_txs_with_empty_meta(&self, _chain: &Chain) -> MmResult, Self::Error> { todo!() } + async fn get_txs_with_empty_meta(&self, chain: &Chain) -> MmResult, Self::Error> { + let locked_db = self.lock_db().await?; + let db_transaction = locked_db.get_inner().transaction().await?; + let table = db_transaction.table::().await?; + let mut cursor = table + .cursor_builder() + .only("chain", chain.to_string()) + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .open_cursor("chain") + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; + + let mut res = HashSet::new(); + while let Some((_item_id, item)) = cursor + .next() + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + { + if item.collection_name.is_none() && item.image.is_none() && item.token_name.is_none() { + res.insert(NftTokenAddrId { + token_address: item.token_address, + token_id: BigDecimal::from_str(&item.token_id).map_err(WasmNftCacheError::ParseBigDecimalError)?, + }); + } + } + drop_mutability!(res); + Ok(res.into_iter().collect()) + } } #[derive(Clone, Debug, Deserialize, Serialize)] @@ -456,7 +588,6 @@ impl NftTxHistoryTable { const CHAIN_TX_HASH_INDEX: &str = "chain_tx_hash_index"; - #[allow(dead_code)] fn from_tx_history(tx: &NftTransferHistory) -> WasmNftCacheResult { let details_json = json::to_value(tx).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftTxHistoryTable { @@ -520,27 +651,3 @@ fn nft_details_from_item(item: NftListTable) -> WasmNftCacheResult { fn tx_details_from_item(item: NftTxHistoryTable) -> WasmNftCacheResult { json::from_value(item.details_json).map_to_mm(|e| WasmNftCacheError::ErrorDeserializing(e.to_string())) } - -fn compare_nft_block_numbers(a: &Nft, b: &Nft) -> Ordering { - let a = BlockNumber::new(a.block_number); - let b = BlockNumber::new(b.block_number); - compare_nfts(a, b) -} - -struct BlockNumber { - block_number: u64, -} - -impl BlockNumber { - fn new(block_number: u64) -> BlockNumber { BlockNumber { block_number } } -} - -fn compare_nfts(a: BlockNumber, b: BlockNumber) -> Ordering { - if a.block_number == 0 { - Ordering::Less - } else if b.block_number == 0 { - Ordering::Greater - } else { - b.block_number.cmp(&a.block_number) - } -} From 016482df91a291cd66bc054bd3f1fb0a7c69d0d8 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 14 Jun 2023 17:06:20 +0700 Subject: [PATCH 071/112] make contract_type mandatory in NftListTable --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 9f6a5c6639..b2e1949151 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -525,7 +525,7 @@ pub(crate) struct NftListTable { chain: String, amount: String, block_number: u64, - contract_type: Option, + contract_type: ContractType, details_json: Json, } From 400e445a45927662ef045afdded45c3659e3615c Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 17:36:28 +0700 Subject: [PATCH 072/112] add test_add_nfts --- mm2src/coins/nft/nft_structs.rs | 6 +- mm2src/coins/nft/nft_tests.rs | 208 ++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 42038abe79..c546a7ca79 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -133,9 +133,9 @@ pub(crate) struct UriMeta { pub(crate) image: Option, #[serde(rename(deserialize = "name"))] pub(crate) token_name: Option, - description: Option, - attributes: Option, - animation_url: Option, + pub(crate) description: Option, + pub(crate) attributes: Option, + pub(crate) animation_url: Option, } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 1cbdacef24..dd410c7983 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -3,9 +3,210 @@ const NFT_HISTORY_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/0x const NFT_METADATA_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal"; const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; +#[cfg(any(test, target_arch = "wasm32"))] +mod for_db_tests { + use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, TransferStatus, UriMeta}; + use crate::nft::storage::{NftListStorageOps, NftStorageBuilder}; + use mm2_number::BigDecimal; + use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; + use std::str::FromStr; + + cfg_wasm32! { + use wasm_bindgen_test::*; + + wasm_bindgen_test_configure!(run_in_browser); + } + + fn nft_list() -> Vec { + let nft = Nft { + chain: Chain::Bsc, + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "b34ddf294013d20a6d70691027625839".to_string(), + block_number_minted: 25465916, + block_number: 25919780, + contract_type: ContractType::Erc1155, + collection_name: None, + symbol: None, + token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + metadata: Some("{\"name\":\"Tiki box\"}".to_string()), + last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), + last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), + minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + token_name: None, + description: Some("Born to usher in Bull markets.".to_string()), + attributes: None, + animation_url: None, + }, + }; + + let nft1 = Nft { + chain: Chain::Bsc, + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "c5d1cfd75a0535b0ec750c0156e6ddfe".to_string(), + block_number_minted: 25721963, + block_number: 28056726, + contract_type: ContractType::Erc721, + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + symbol: Some("BMBBBF".to_string()), + token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), + metadata: Some( + "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), + last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), + minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some( + "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), + ), + token_name: Some("Nebula Nodes".to_string()), + description: Some("Interchain nodes".to_string()), + attributes: None, + animation_url: None, + }, + }; + + let nft2 = Nft { + chain: Chain::Bsc, + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300044414").unwrap(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "125f8f4e952e107c257960000b4b250e".to_string(), + block_number_minted: 25810308, + block_number: 28056721, + contract_type: ContractType::Erc721, + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + symbol: Some("BMBBBF".to_string()), + token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300044414".to_string()), + metadata: Some( + "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-19T19:12:09.471Z".to_string()), + last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), + minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some( + "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), + ), + token_name: Some("Nebula Nodes".to_string()), + description: Some("Interchain nodes".to_string()), + attributes: None, + animation_url: None, + }, + }; + vec![nft, nft1, nft2] + } + + #[allow(dead_code)] + fn nft_tx_historty() -> Vec { + let tx = NftTransferHistory { + chain: Chain::Bsc, + block_number: 25919780, + block_timestamp: 1677166110, + block_hash: "0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string(), + transaction_hash: "0x9c16b962f63eead1c5d2355cc9037dde178b14b53043c57eb40c27964d22ae6a".to_string(), + transaction_index: 57, + log_index: 139, + value: Default::default(), + contract_type: ContractType::Erc1155, + transaction_type: "Single".to_string(), + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + collection_name: None, + image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + token_name: None, + from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), + possible_spam: Some(false), + }; + + let tx1 = NftTransferHistory { + chain: Chain::Bsc, + block_number: 28056726, + block_timestamp: 1683627432, + block_hash: "0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string(), + transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), + transaction_index: 198, + log_index: 495, + value: Default::default(), + contract_type: ContractType::Erc721, + transaction_type: "Single".to_string(), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: None, + possible_spam: Some(false), + }; + + let tx2 = NftTransferHistory { + chain: Chain::Bsc, + block_number: 28056721, + block_timestamp: 1683627417, + block_hash: "0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string(), + transaction_hash: "0x981bad702cc6e088f0e9b5e7287ff7a3487b8d269103cee3b9e5803141f63f91".to_string(), + transaction_index: 83, + log_index: 201, + value: Default::default(), + contract_type: ContractType::Erc721, + transaction_type: "Single".to_string(), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300044414").unwrap(), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: None, + possible_spam: Some(false), + }; + vec![tx, tx1, tx2] + } + + pub(crate) async fn test_add_nfts_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(!is_initialized); + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let scanned_block = 28056726; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + } +} + #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; + use crate::nft::nft_tests::for_db_tests::*; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::send_request_to_uri; use common::block_on; @@ -39,11 +240,15 @@ mod native_tests { let uri_response = block_on(send_request_to_uri(token_uri.as_str())).unwrap(); serde_json::from_str::(&uri_response.to_string()).unwrap(); } + + #[test] + fn test_add_nfts() { block_on(test_add_nfts_impl()) } } #[cfg(target_arch = "wasm32")] mod wasm_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper}; + use crate::nft::nft_tests::for_db_tests::*; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::send_request_to_uri; use wasm_bindgen_test::*; @@ -76,4 +281,7 @@ mod wasm_tests { let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); assert_eq!(41237364, *nft_wrapper.block_number_minted); } + + #[wasm_bindgen_test] + async fn test_add_nfts() { test_add_nfts_impl().await } } From f4fcc9bec64ac3035e03839337ed07ee1f4f4ed2 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 17:59:27 +0700 Subject: [PATCH 073/112] initialize db first --- mm2src/coins/nft/nft_tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index dd410c7983..23adc73bba 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -194,9 +194,9 @@ mod for_db_tests { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(!is_initialized); NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); let scanned_block = 28056726; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); From c49fd354b60ddf870accd351bf9d2548e90f4d5f Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 18:35:51 +0700 Subject: [PATCH 074/112] last blocks in tests --- mm2src/coins/nft/nft_tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 23adc73bba..1bc7a6562b 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -200,6 +200,9 @@ mod for_db_tests { let scanned_block = 28056726; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + let last_block = storage.get_last_block_number(&chain).await.unwrap().unwrap(); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_block, last_scanned_block); } } From 2697d0a7c15e51d653f34b9399f715078a8a3e22 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 18:54:36 +0700 Subject: [PATCH 075/112] remove CHAIN_BLOCK_NUMBER_INDEX --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index b2e1949151..068cc6c53b 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -242,7 +242,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .bound("block_number", 0u64, u64::MAX) .reverse() - .open_cursor(NftListTable::CHAIN_BLOCK_NUMBER_INDEX) + .open_cursor("chain") .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .next() @@ -530,8 +530,6 @@ pub(crate) struct NftListTable { } impl NftListTable { - const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; - const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; fn from_nft(nft: &Nft) -> WasmNftCacheResult { @@ -554,7 +552,6 @@ impl TableSignature for NftListTable { fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { if let (0, 1) = (old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; - table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; table.create_multi_index( Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, &["chain", "token_address", "token_id"], From 3f090713ffe4fd6d7acd08bd3017dfc332d2afea Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 19:54:53 +0700 Subject: [PATCH 076/112] check get_nft --- mm2src/coins/nft/nft_tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 1bc7a6562b..7bd33cb7ea 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -200,8 +200,12 @@ mod for_db_tests { let scanned_block = 28056726; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); - let last_block = storage.get_last_block_number(&chain).await.unwrap().unwrap(); + let token_add = "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(); + let token_id = BigDecimal::from_str("214300044414").unwrap(); + let nft = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(nft.block_number, 28056721); let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + let last_block = storage.get_last_block_number(&chain).await.unwrap().unwrap(); assert_eq!(last_block, last_scanned_block); } } From 70d1de48f58c02013675f30392fbbcc99a093fed Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 22:33:11 +0700 Subject: [PATCH 077/112] add txs test --- mm2src/coins/nft/nft_tests.rs | 43 +++++++++++++++++-- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 13 ++++-- 2 files changed, 49 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 7bd33cb7ea..138c097201 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -6,7 +6,7 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; #[cfg(any(test, target_arch = "wasm32"))] mod for_db_tests { use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, TransferStatus, UriMeta}; - use crate::nft::storage::{NftListStorageOps, NftStorageBuilder}; + use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; use std::str::FromStr; @@ -111,7 +111,6 @@ mod for_db_tests { vec![nft, nft1, nft2] } - #[allow(dead_code)] fn nft_tx_historty() -> Vec { let tx = NftTransferHistory { chain: Chain::Bsc, @@ -204,10 +203,42 @@ mod for_db_tests { let token_id = BigDecimal::from_str("214300044414").unwrap(); let nft = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); assert_eq!(nft.block_number, 28056721); + let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - let last_block = storage.get_last_block_number(&chain).await.unwrap().unwrap(); assert_eq!(last_block, last_scanned_block); } + + pub(crate) async fn test_add_txs_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + let token_add = "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(); + let token_id = BigDecimal::from_str("214300044414").unwrap(); + let tx1 = storage + .get_txs_by_token_addr_id(&chain, token_add, token_id) + .await + .unwrap() + .get(0) + .unwrap() + .clone(); + assert_eq!(tx1.block_number, 28056721); + let tx_hash = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(); + let tx2 = storage.get_tx_by_tx_hash(&chain, tx_hash).await.unwrap().unwrap(); + assert_eq!(tx2.block_number, 28056726); + let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); + } } #[cfg(all(test, not(target_arch = "wasm32")))] @@ -250,6 +281,9 @@ mod native_tests { #[test] fn test_add_nfts() { block_on(test_add_nfts_impl()) } + + #[test] + fn test_add_txs() { block_on(test_add_txs_impl()) } } #[cfg(target_arch = "wasm32")] @@ -291,4 +325,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_add_nfts() { test_add_nfts_impl().await } + + #[wasm_bindgen_test] + async fn test_add_txs() { test_add_txs_impl().await } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 068cc6c53b..ac889fb96c 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -242,7 +242,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .bound("block_number", 0u64, u64::MAX) .reverse() - .open_cursor("chain") + .open_cursor(NftListTable::CHAIN_BLOCK_NUMBER_INDEX) .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .next() @@ -376,7 +376,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .bound("block_number", 0u64, u64::MAX) .reverse() - .open_cursor("chain") + .open_cursor("block_number") .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .next() @@ -398,7 +398,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .only("chain", chain.to_string()) .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .bound("block_number", from_block, u64::MAX) - .open_cursor("chain") + .open_cursor(NftTxHistoryTable::CHAIN_BLOCK_NUMBER_INDEX) .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; @@ -427,7 +427,6 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? - .with_value(&token_address)? .with_value(token_id.to_string())?; table .get_items_by_multi_index(index_keys) @@ -532,6 +531,8 @@ pub(crate) struct NftListTable { impl NftListTable { const CHAIN_TOKEN_ADD_TOKEN_ID_INDEX: &str = "chain_token_add_token_id_index"; + const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; + fn from_nft(nft: &Nft) -> WasmNftCacheResult { let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftListTable { @@ -557,6 +558,7 @@ impl TableSignature for NftListTable { &["chain", "token_address", "token_id"], true, )?; + table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], true)?; table.create_index("chain", false)?; } Ok(()) @@ -585,6 +587,8 @@ impl NftTxHistoryTable { const CHAIN_TX_HASH_INDEX: &str = "chain_tx_hash_index"; + const CHAIN_BLOCK_NUMBER_INDEX: &str = "chain_block_number_index"; + fn from_tx_history(tx: &NftTransferHistory) -> WasmNftCacheResult { let details_json = json::to_value(tx).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftTxHistoryTable { @@ -617,6 +621,7 @@ impl TableSignature for NftTxHistoryTable { false, )?; table.create_multi_index(Self::CHAIN_TX_HASH_INDEX, &["chain", "transaction_hash"], true)?; + table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], true)?; table.create_index("chain", false)?; } Ok(()) From cd64d40be8acbf484827b640205ce849cb34f3d5 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 23:04:59 +0700 Subject: [PATCH 078/112] add get_txs_from_block in tests --- mm2src/coins/nft/nft_tests.rs | 2 ++ mm2src/coins/nft/storage/wasm/wasm_storage.rs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 138c097201..a32f455dee 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -233,6 +233,8 @@ mod for_db_tests { let tx_hash = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(); let tx2 = storage.get_tx_by_tx_hash(&chain, tx_hash).await.unwrap().unwrap(); assert_eq!(tx2.block_number, 28056726); + let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); + assert_eq!(tx_from.len(), 2); let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) .await .unwrap() diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index ac889fb96c..2bb99a3c13 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -376,7 +376,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .bound("block_number", 0u64, u64::MAX) .reverse() - .open_cursor("block_number") + .open_cursor(NftTxHistoryTable::CHAIN_BLOCK_NUMBER_INDEX) .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? .next() From 590de5f63cb21a667dd7e3c4217b1b73c47a204c Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 23:14:45 +0700 Subject: [PATCH 079/112] get_item_by_unique_index in get last scanned block --- mm2src/coins/nft/nft_tests.rs | 4 ++-- mm2src/coins/nft/storage/wasm/mod.rs | 1 - mm2src/coins/nft/storage/wasm/wasm_storage.rs | 18 +++++------------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index a32f455dee..4357551368 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -203,12 +203,12 @@ mod for_db_tests { let token_id = BigDecimal::from_str("214300044414").unwrap(); let nft = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); assert_eq!(nft.block_number, 28056721); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) .await .unwrap() .unwrap(); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_block, last_scanned_block); + assert_eq!(last_scanned_block, last_block); } pub(crate) async fn test_add_txs_impl() { diff --git a/mm2src/coins/nft/storage/wasm/mod.rs b/mm2src/coins/nft/storage/wasm/mod.rs index dd4e58568b..c7d5cd7a1e 100644 --- a/mm2src/coins/nft/storage/wasm/mod.rs +++ b/mm2src/coins/nft/storage/wasm/mod.rs @@ -20,7 +20,6 @@ pub enum WasmNftCacheError { NotSupported(String), InternalError(String), GetLastNftBlockError(String), - GetLastScannedBlockError(String), ParseBigDecimalError(ParseBigDecimalError), } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 2bb99a3c13..9887865af8 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -255,19 +255,11 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let maybe_item = table - .cursor_builder() - .only("chain", chain.to_string()) - .map_err(|e| WasmNftCacheError::GetLastScannedBlockError(e.to_string()))? - .bound("last_scanned_block", 0u64, u64::MAX) - .reverse() - .open_cursor("chain") - .await - .map_err(|e| WasmNftCacheError::GetLastScannedBlockError(e.to_string()))? - .next() - .await - .map_err(|e| WasmNftCacheError::GetLastScannedBlockError(e.to_string()))?; - Ok(maybe_item.map(|(_, item)| item.last_scanned_block)) + if let Some((_item_id, item)) = table.get_item_by_unique_index("chain", chain.to_string()).await? { + Ok(Some(item.last_scanned_block)) + } else { + return Ok(None); + } } async fn update_nft_amount(&self, chain: &Chain, nft: Nft, scanned_block: u64) -> MmResult<(), Self::Error> { From 114305778547252af013fc504c7711359995e28d Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 15 Jun 2023 23:39:45 +0700 Subject: [PATCH 080/112] create block_number index --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 9887865af8..22bed771f2 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -550,8 +550,9 @@ impl TableSignature for NftListTable { &["chain", "token_address", "token_id"], true, )?; - table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], true)?; + table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; table.create_index("chain", false)?; + table.create_index("block_number", false)?; } Ok(()) } @@ -614,6 +615,7 @@ impl TableSignature for NftTxHistoryTable { )?; table.create_multi_index(Self::CHAIN_TX_HASH_INDEX, &["chain", "transaction_hash"], true)?; table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], true)?; + table.create_index("block_number", true)?; table.create_index("chain", false)?; } Ok(()) From c33a014981d8a72698d5b38eda82414c807c638f Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 16 Jun 2023 11:22:59 +0700 Subject: [PATCH 081/112] use BeBigUint --- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 60 ++++++++++++------- 1 file changed, 40 insertions(+), 20 deletions(-) diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 22bed771f2..6bd8b80161 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -7,11 +7,12 @@ use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorag use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; -use mm2_db::indexed_db::{DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; +use mm2_db::indexed_db::{BeBigUint, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; use mm2_number::BigDecimal; +use num_traits::ToPrimitive; use serde_json::{self as json, Value as Json}; use std::collections::HashSet; use std::num::NonZeroUsize; @@ -138,7 +139,7 @@ impl NftListStorageOps for IndexedDbNftStorage { } let last_scanned_block = LastScannedBlockTable { chain: chain.to_string(), - last_scanned_block, + last_scanned_block: BeBigUint::from(last_scanned_block), }; last_scanned_block_table .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) @@ -184,7 +185,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(token_id.to_string())?; let last_scanned_block = LastScannedBlockTable { chain: chain.to_string(), - last_scanned_block: scanned_block, + last_scanned_block: BeBigUint::from(scanned_block), }; let nft_removed = nft_table.delete_item_by_unique_multi_index(index_keys).await?.is_some(); @@ -240,7 +241,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .cursor_builder() .only("chain", chain.to_string()) .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .bound("block_number", 0u64, u64::MAX) + .bound("block_number", BeBigUint::from(0u64), BeBigUint::from(u64::MAX)) .reverse() .open_cursor(NftListTable::CHAIN_BLOCK_NUMBER_INDEX) .await @@ -248,7 +249,15 @@ impl NftListStorageOps for IndexedDbNftStorage { .next() .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; - Ok(maybe_item.map(|(_, item)| item.block_number)) + let maybe_item = maybe_item + .map(|(_, item)| { + item.block_number + .to_u64() + .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string())) + }) + .transpose()?; + + Ok(maybe_item) } async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { @@ -256,7 +265,11 @@ impl NftListStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; if let Some((_item_id, item)) = table.get_item_by_unique_index("chain", chain.to_string()).await? { - Ok(Some(item.last_scanned_block)) + let last_scanned_block = item + .last_scanned_block + .to_u64() + .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string()))?; + Ok(Some(last_scanned_block)) } else { return Ok(None); } @@ -278,7 +291,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .await?; let last_scanned_block = LastScannedBlockTable { chain: chain.to_string(), - last_scanned_block: scanned_block, + last_scanned_block: BeBigUint::from(scanned_block), }; last_scanned_block_table .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) @@ -302,7 +315,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .await?; let last_scanned_block = LastScannedBlockTable { chain: chain.to_string(), - last_scanned_block: nft.block_number, + last_scanned_block: BeBigUint::from(nft.block_number), }; last_scanned_block_table .replace_item_by_unique_index("chain", chain.to_string(), &last_scanned_block) @@ -366,7 +379,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .cursor_builder() .only("chain", chain.to_string()) .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .bound("block_number", 0u64, u64::MAX) + .bound("block_number", BeBigUint::from(0u64), BeBigUint::from(u64::MAX)) .reverse() .open_cursor(NftTxHistoryTable::CHAIN_BLOCK_NUMBER_INDEX) .await @@ -374,7 +387,14 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .next() .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; - Ok(maybe_item.map(|(_, item)| item.block_number)) + let maybe_item = maybe_item + .map(|(_, item)| { + item.block_number + .to_u64() + .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string())) + }) + .transpose()?; + Ok(maybe_item) } async fn get_txs_from_block( @@ -389,7 +409,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .cursor_builder() .only("chain", chain.to_string()) .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .bound("block_number", from_block, u64::MAX) + .bound("block_number", BeBigUint::from(from_block), BeBigUint::from(u64::MAX)) .open_cursor(NftTxHistoryTable::CHAIN_BLOCK_NUMBER_INDEX) .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; @@ -515,7 +535,7 @@ pub(crate) struct NftListTable { token_id: String, chain: String, amount: String, - block_number: u64, + block_number: BeBigUint, contract_type: ContractType, details_json: Json, } @@ -532,7 +552,7 @@ impl NftListTable { token_id: nft.token_id.to_string(), chain: nft.chain.to_string(), amount: nft.amount.to_string(), - block_number: nft.block_number, + block_number: BeBigUint::from(nft.block_number), contract_type: nft.contract_type, details_json, }) @@ -562,8 +582,8 @@ impl TableSignature for NftListTable { pub(crate) struct NftTxHistoryTable { transaction_hash: String, chain: String, - block_number: u64, - block_timestamp: u64, + block_number: BeBigUint, + block_timestamp: BeBigUint, contract_type: ContractType, token_address: String, token_id: String, @@ -587,8 +607,8 @@ impl NftTxHistoryTable { Ok(NftTxHistoryTable { transaction_hash: tx.transaction_hash.clone(), chain: tx.chain.to_string(), - block_number: tx.block_number, - block_timestamp: tx.block_timestamp, + block_number: BeBigUint::from(tx.block_number), + block_timestamp: BeBigUint::from(tx.block_timestamp), contract_type: tx.contract_type, token_address: tx.token_address.clone(), token_id: tx.token_id.to_string(), @@ -614,8 +634,8 @@ impl TableSignature for NftTxHistoryTable { false, )?; table.create_multi_index(Self::CHAIN_TX_HASH_INDEX, &["chain", "transaction_hash"], true)?; - table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], true)?; - table.create_index("block_number", true)?; + table.create_multi_index(Self::CHAIN_BLOCK_NUMBER_INDEX, &["chain", "block_number"], false)?; + table.create_index("block_number", false)?; table.create_index("chain", false)?; } Ok(()) @@ -625,7 +645,7 @@ impl TableSignature for NftTxHistoryTable { #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct LastScannedBlockTable { chain: String, - last_scanned_block: u64, + last_scanned_block: BeBigUint, } impl TableSignature for LastScannedBlockTable { From 5ca293300081e0d54b4cc10e9bc6798324a7b268 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 16 Jun 2023 14:42:54 +0700 Subject: [PATCH 082/112] match contract_type, NOT_FOUND contract_type --- mm2src/coins/nft.rs | 125 +++++++++++++++++---------------- mm2src/coins/nft/nft_errors.rs | 7 +- 2 files changed, 69 insertions(+), 63 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 1c631db638..430e0b079f 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -205,31 +205,33 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult if let Some(nfts_list) = response["result"].as_array() { for nft_json in nfts_list { let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; - if nft_wrapper.contract_type.is_some() { - let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; - let nft = Nft { - chain: *chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, - block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.unwrap().0, - collection_name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, - uri_meta, - }; - // collect NFTs from the page - res_list.push(nft); - } + let contract_type = match nft_wrapper.contract_type { + Some(contract_type) => contract_type.0, + None => continue, + }; + let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; + let nft = Nft { + chain: *chain, + token_address: nft_wrapper.token_address, + token_id: nft_wrapper.token_id.0, + amount: nft_wrapper.amount.0, + owner_of: nft_wrapper.owner_of, + token_hash: nft_wrapper.token_hash, + block_number_minted: *nft_wrapper.block_number_minted, + block_number: *nft_wrapper.block_number, + contract_type, + collection_name: nft_wrapper.name, + symbol: nft_wrapper.symbol, + token_uri: nft_wrapper.token_uri, + metadata: nft_wrapper.metadata, + last_token_uri_sync: nft_wrapper.last_token_uri_sync, + last_metadata_sync: nft_wrapper.last_metadata_sync, + minter_address: nft_wrapper.minter_address, + possible_spam: nft_wrapper.possible_spam, + uri_meta, + }; + // collect NFTs from the page + res_list.push(nft); } // if cursor is not null, there are other NFTs on next page, // and we need to send new request with cursor to get info from the next page. @@ -283,37 +285,39 @@ async fn get_moralis_nft_transfers( if let Some(transfer_list) = response["result"].as_array() { for transfer in transfer_list { let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; - if transfer_wrapper.contract_type.is_some() { - let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); - let block_timestamp = parse_rfc3339_to_timestamp(&transfer_wrapper.block_timestamp) - .map_to_mm(|e| GetNftInfoError::ParseTimestampError(e.to_string()))?; - let transfer_history = NftTransferHistory { - chain: *chain, - block_number: *transfer_wrapper.block_number, - block_timestamp, - block_hash: transfer_wrapper.block_hash, - transaction_hash: transfer_wrapper.transaction_hash, - transaction_index: transfer_wrapper.transaction_index, - log_index: transfer_wrapper.log_index, - value: transfer_wrapper.value.0, - contract_type: transfer_wrapper.contract_type.unwrap().0, - transaction_type: transfer_wrapper.transaction_type, - token_address: transfer_wrapper.token_address, - token_id: transfer_wrapper.token_id.0, - collection_name: None, - image: None, - token_name: None, - from_address: transfer_wrapper.from_address, - to_address: transfer_wrapper.to_address, - status, - amount: transfer_wrapper.amount.0, - verified: transfer_wrapper.verified, - operator: transfer_wrapper.operator, - possible_spam: transfer_wrapper.possible_spam, - }; - // collect NFTs transfers from the page - res_list.push(transfer_history); - } + let contract_type = match transfer_wrapper.contract_type { + Some(contract_type) => contract_type.0, + None => continue, + }; + let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); + let block_timestamp = parse_rfc3339_to_timestamp(&transfer_wrapper.block_timestamp) + .map_to_mm(|e| GetNftInfoError::ParseTimestampError(e.to_string()))?; + let transfer_history = NftTransferHistory { + chain: *chain, + block_number: *transfer_wrapper.block_number, + block_timestamp, + block_hash: transfer_wrapper.block_hash, + transaction_hash: transfer_wrapper.transaction_hash, + transaction_index: transfer_wrapper.transaction_index, + log_index: transfer_wrapper.log_index, + value: transfer_wrapper.value.0, + contract_type, + transaction_type: transfer_wrapper.transaction_type, + token_address: transfer_wrapper.token_address, + token_id: transfer_wrapper.token_id.0, + collection_name: None, + image: None, + token_name: None, + from_address: transfer_wrapper.from_address, + to_address: transfer_wrapper.to_address, + status, + amount: transfer_wrapper.amount.0, + verified: transfer_wrapper.verified, + operator: transfer_wrapper.operator, + possible_spam: transfer_wrapper.possible_spam, + }; + // collect NFTs transfers from the page + res_list.push(transfer_history); } // if the cursor is not null, there are other NFTs transfers on next page, // and we need to send new request with cursor to get info from the next page. @@ -352,9 +356,10 @@ async fn get_moralis_metadata( let response = send_request_to_uri(uri.as_str()).await?; let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string())?; - if nft_wrapper.contract_type.is_none() { - return MmError::err(GetNftInfoError::ContractTypeIsNull); - } + let contract_type = match nft_wrapper.contract_type { + Some(contract_type) => contract_type.0, + None => return MmError::err(GetNftInfoError::ContractTypeIsNull), + }; let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; let nft_metadata = Nft { chain: *chain, @@ -365,7 +370,7 @@ async fn get_moralis_metadata( token_hash: nft_wrapper.token_hash, block_number_minted: *nft_wrapper.block_number_minted, block_number: *nft_wrapper.block_number, - contract_type: nft_wrapper.contract_type.unwrap().0, + contract_type, collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, token_uri: nft_wrapper.token_uri, diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index a5defb9beb..3b4db40556 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -104,9 +104,10 @@ impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { GetNftInfoError::InvalidRequest(_) => StatusCode::BAD_REQUEST, - GetNftInfoError::InvalidResponse(_) - | GetNftInfoError::ParseTimestampError(_) - | GetNftInfoError::ContractTypeIsNull => StatusCode::FAILED_DEPENDENCY, + GetNftInfoError::InvalidResponse(_) | GetNftInfoError::ParseTimestampError(_) => { + StatusCode::FAILED_DEPENDENCY + }, + GetNftInfoError::ContractTypeIsNull => StatusCode::NOT_FOUND, GetNftInfoError::Transport(_) | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) From 59c0db5392170e74627bc6bb6fbba04838cd93c5 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 16 Jun 2023 16:44:17 +0700 Subject: [PATCH 083/112] separate last block tests --- mm2src/coins/nft/nft_tests.rs | 75 ++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 4357551368..c259bd06b6 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -17,6 +17,10 @@ mod for_db_tests { wasm_bindgen_test_configure!(run_in_browser); } + const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; + const TOKEN_ID: &str = "214300044414"; + const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; + fn nft_list() -> Vec { let nft = Nft { chain: Chain::Bsc, @@ -189,7 +193,7 @@ mod for_db_tests { vec![tx, tx1, tx2] } - pub(crate) async fn test_add_nfts_impl() { + pub(crate) async fn test_add_get_nfts_impl() { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; @@ -199,10 +203,27 @@ mod for_db_tests { let scanned_block = 28056726; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); - let token_add = "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(); - let token_id = BigDecimal::from_str("214300044414").unwrap(); - let nft = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); assert_eq!(nft.block_number, 28056721); + } + + pub(crate) async fn test_last_nft_blocks_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let scanned_block = 28056726; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) .await @@ -211,7 +232,7 @@ mod for_db_tests { assert_eq!(last_scanned_block, last_block); } - pub(crate) async fn test_add_txs_impl() { + pub(crate) async fn test_add_get_txs_impl() { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; @@ -220,21 +241,35 @@ mod for_db_tests { assert!(is_initialized); let txs = nft_tx_historty(); storage.add_txs_to_history(&chain, txs).await.unwrap(); - let token_add = "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(); - let token_id = BigDecimal::from_str("214300044414").unwrap(); + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); let tx1 = storage - .get_txs_by_token_addr_id(&chain, token_add, token_id) + .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) .await .unwrap() .get(0) .unwrap() .clone(); assert_eq!(tx1.block_number, 28056721); - let tx_hash = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(); - let tx2 = storage.get_tx_by_tx_hash(&chain, tx_hash).await.unwrap().unwrap(); + let tx2 = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); assert_eq!(tx2.block_number, 28056726); let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); assert_eq!(tx_from.len(), 2); + } + + pub(crate) async fn test_last_tx_block_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) .await .unwrap() @@ -282,10 +317,16 @@ mod native_tests { } #[test] - fn test_add_nfts() { block_on(test_add_nfts_impl()) } + fn test_add_get_nfts() { block_on(test_add_get_nfts_impl()) } + + #[test] + fn test_last_nft_blocks() { block_on(test_last_nft_blocks_impl()) } + + #[test] + fn test_add_get_txs() { block_on(test_add_get_txs_impl()) } #[test] - fn test_add_txs() { block_on(test_add_txs_impl()) } + fn test_last_tx_block() { block_on(test_last_tx_block_impl()) } } #[cfg(target_arch = "wasm32")] @@ -326,8 +367,14 @@ mod wasm_tests { } #[wasm_bindgen_test] - async fn test_add_nfts() { test_add_nfts_impl().await } + async fn test_add_get_nfts() { test_add_get_nfts_impl().await } + + #[wasm_bindgen_test] + async fn test_last_nft_blocks() { test_last_nft_blocks_impl().await } + + #[wasm_bindgen_test] + async fn test_add_get_txs() { test_add_get_txs_impl().await } #[wasm_bindgen_test] - async fn test_add_txs() { test_add_txs_impl().await } + async fn test_last_tx_block() { test_last_tx_block_impl().await } } From f3e0776448d09426a76b6b665c79081d7cd0f814 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 16 Jun 2023 17:10:14 +0700 Subject: [PATCH 084/112] test_nft_list --- mm2src/coins/nft/nft_tests.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index c259bd06b6..9ba0d195af 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -9,6 +9,7 @@ mod for_db_tests { use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; + use std::num::NonZeroUsize; use std::str::FromStr; cfg_wasm32! { @@ -232,6 +233,28 @@ mod for_db_tests { assert_eq!(last_scanned_block, last_block); } + pub(crate) async fn test_nft_list_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let scanned_block = 28056726; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + + let nft_list = storage + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 1); + let nft = nft_list.nfts.get(0).unwrap(); + assert_eq!(nft.block_number, 28056721); + assert_eq!(nft_list.skipped, 1); + assert_eq!(nft_list.total, 3); + } + pub(crate) async fn test_add_get_txs_impl() { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); @@ -322,6 +345,9 @@ mod native_tests { #[test] fn test_last_nft_blocks() { block_on(test_last_nft_blocks_impl()) } + #[test] + fn test_nft_list() { block_on(test_nft_list_impl()) } + #[test] fn test_add_get_txs() { block_on(test_add_get_txs_impl()) } @@ -372,6 +398,9 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_last_nft_blocks() { test_last_nft_blocks_impl().await } + #[wasm_bindgen_test] + async fn test_nft_list() { test_nft_list_impl().await } + #[wasm_bindgen_test] async fn test_add_get_txs() { test_add_get_txs_impl().await } From b5bd19bc6e45fe173f051b0e7aa603f216495f49 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 16 Jun 2023 18:17:35 +0700 Subject: [PATCH 085/112] test remove nft and update amount --- mm2src/coins/nft/nft_tests.rs | 121 +++++++++++++++++++++++++++++--- mm2src/coins/nft/storage/mod.rs | 3 +- 2 files changed, 114 insertions(+), 10 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 9ba0d195af..d642af6d2d 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -6,7 +6,7 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; #[cfg(any(test, target_arch = "wasm32"))] mod for_db_tests { use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, TransferStatus, UriMeta}; - use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; + use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; use std::num::NonZeroUsize; @@ -22,12 +22,41 @@ mod for_db_tests { const TOKEN_ID: &str = "214300044414"; const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; + fn nft() -> Nft { + Nft { + chain: Chain::Bsc, + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + amount: BigDecimal::from_str("2").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "b34ddf294013d20a6d70691027625839".to_string(), + block_number_minted: 25465916, + block_number: 25919780, + contract_type: ContractType::Erc1155, + collection_name: None, + symbol: None, + token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + metadata: Some("{\"name\":\"Tiki box\"}".to_string()), + last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), + last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), + minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + token_name: None, + description: Some("Born to usher in Bull markets.".to_string()), + attributes: None, + animation_url: None, + }, + } + } + fn nft_list() -> Vec { let nft = Nft { chain: Chain::Bsc, token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), token_id: Default::default(), - amount: BigDecimal::from_str("1").unwrap(), + amount: BigDecimal::from_str("2").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), token_hash: "b34ddf294013d20a6d70691027625839".to_string(), block_number_minted: 25465916, @@ -201,9 +230,8 @@ mod for_db_tests { NftListStorageOps::init(&storage, &chain).await.unwrap(); let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); assert!(is_initialized); - let scanned_block = 28056726; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); let nft = storage @@ -221,9 +249,8 @@ mod for_db_tests { NftListStorageOps::init(&storage, &chain).await.unwrap(); let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); assert!(is_initialized); - let scanned_block = 28056726; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) @@ -240,9 +267,8 @@ mod for_db_tests { NftListStorageOps::init(&storage, &chain).await.unwrap(); let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); assert!(is_initialized); - let scanned_block = 28056726; let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, scanned_block).await.unwrap(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); let nft_list = storage .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) @@ -255,6 +281,73 @@ mod for_db_tests { assert_eq!(nft_list.total, 3); } + pub(crate) async fn test_remove_nft_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let remove_rslt = storage + .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) + .await + .unwrap(); + assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); + let list_len = storage + .get_nft_list(vec![chain], true, 10, None) + .await + .unwrap() + .nfts + .len(); + assert_eq!(list_len, 2); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 28056800); + } + + pub(crate) async fn test_nft_amount_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + + nft.amount -= BigDecimal::from(1); + storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "1"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919800); + + nft.amount += BigDecimal::from(1); + nft.block_number = 25919900; + storage + .update_nft_amount_and_block_number(&chain, nft.clone()) + .await + .unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address, nft.token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "2"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919900); + } + pub(crate) async fn test_add_get_txs_impl() { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); @@ -348,6 +441,12 @@ mod native_tests { #[test] fn test_nft_list() { block_on(test_nft_list_impl()) } + #[test] + fn test_remove_nft() { block_on(test_remove_nft_impl()) } + + #[test] + fn test_nft_amount() { block_on(test_nft_amount_impl()) } + #[test] fn test_add_get_txs() { block_on(test_add_get_txs_impl()) } @@ -401,6 +500,12 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_nft_list() { test_nft_list_impl().await } + #[wasm_bindgen_test] + async fn test_remove_nft() { test_remove_nft_impl().await } + + #[wasm_bindgen_test] + async fn test_nft_amount() { test_nft_amount_impl().await } + #[wasm_bindgen_test] async fn test_add_get_txs() { test_add_get_txs_impl().await } diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index b0cdc8a9b3..11af867f69 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -13,8 +13,7 @@ use std::num::NonZeroUsize; #[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; #[cfg(target_arch = "wasm32")] pub mod wasm; -#[allow(dead_code)] -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum RemoveNftResult { NftRemoved, NftDidNotExist, From e18f932abe0859ea0d5c6a8eaed3ad18537ea149 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 16 Jun 2023 21:48:01 +0700 Subject: [PATCH 086/112] test_refresh_metadata, test_tx_history --- mm2src/coins/nft/nft_tests.rs | 125 +++++++++++++++++- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 22 ++- 2 files changed, 139 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index d642af6d2d..f1f3a0bc77 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -5,7 +5,8 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; #[cfg(any(test, target_arch = "wasm32"))] mod for_db_tests { - use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, TransferStatus, UriMeta}; + use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, + UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; @@ -298,7 +299,7 @@ mod for_db_tests { .unwrap(); assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); let list_len = storage - .get_nft_list(vec![chain], true, 10, None) + .get_nft_list(vec![chain], true, 1, None) .await .unwrap() .nfts @@ -348,6 +349,29 @@ mod for_db_tests { assert_eq!(last_scanned_block, 25919900); } + pub(crate) async fn test_refresh_metadata_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + + let new_symbol = "NEW_SYMBOL"; + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + nft.symbol = Some(new_symbol.to_string()); + drop_mutability!(nft); + let token_add = nft.token_address.clone(); + let token_id = nft.token_id.clone(); + storage.refresh_nft_metadata(&chain, nft).await.unwrap(); + let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); + } + pub(crate) async fn test_add_get_txs_impl() { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); @@ -392,6 +416,85 @@ mod for_db_tests { .unwrap(); assert_eq!(last_block, 28056726); } + + pub(crate) async fn test_tx_history_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let tx_history = storage + .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 1); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056721); + assert_eq!(tx_history.skipped, 1); + assert_eq!(tx_history.total, 3); + } + + pub(crate) async fn test_tx_history_filters_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let filters = NftTxHistoryFilters { + receive: true, + send: false, + from_date: None, + to_date: None, + }; + + let filters1 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: None, + to_date: Some(1677166110), + }; + + let filters2 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: Some(1677166110), + to_date: Some(1683627417), + }; + + let tx_history = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 3); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056726); + + let tx_history1 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap(); + assert_eq!(tx_history1.transfer_history.len(), 1); + let tx1 = tx_history1.transfer_history.get(0).unwrap(); + assert_eq!(tx1.block_number, 25919780); + + let tx_history2 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters2)) + .await + .unwrap(); + assert_eq!(tx_history2.transfer_history.len(), 2); + let tx_0 = tx_history2.transfer_history.get(0).unwrap(); + assert_eq!(tx_0.block_number, 28056721); + let tx_1 = tx_history2.transfer_history.get(1).unwrap(); + assert_eq!(tx_1.block_number, 25919780); + } } #[cfg(all(test, not(target_arch = "wasm32")))] @@ -444,6 +547,9 @@ mod native_tests { #[test] fn test_remove_nft() { block_on(test_remove_nft_impl()) } + #[test] + fn test_refresh_metadata() { block_on(test_refresh_metadata_impl()) } + #[test] fn test_nft_amount() { block_on(test_nft_amount_impl()) } @@ -452,6 +558,12 @@ mod native_tests { #[test] fn test_last_tx_block() { block_on(test_last_tx_block_impl()) } + + #[test] + fn test_tx_history() { block_on(test_tx_history_impl()) } + + #[test] + fn test_tx_history_filters() { block_on(test_tx_history_filters_impl()) } } #[cfg(target_arch = "wasm32")] @@ -506,9 +618,18 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_nft_amount() { test_nft_amount_impl().await } + #[wasm_bindgen_test] + async fn test_refresh_metadata() { test_refresh_metadata_impl().await } + #[wasm_bindgen_test] async fn test_add_get_txs() { test_add_get_txs_impl().await } #[wasm_bindgen_test] async fn test_last_tx_block() { test_last_tx_block_impl().await } + + #[wasm_bindgen_test] + async fn test_tx_history() { test_tx_history_impl().await } + + #[wasm_bindgen_test] + async fn test_tx_history_filters() { test_tx_history_filters_impl().await } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 6bd8b80161..e93d9174cc 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -75,15 +75,10 @@ impl IndexedDbNftStorage { I: Iterator, { let mut filtered_txs = Vec::new(); - for tx_table in txs { let tx = tx_details_from_item(tx_table)?; if let Some(filters) = &filters { - let status_matches = (filters.receive && tx.status == TransferStatus::Receive) - || (filters.send && tx.status == TransferStatus::Send); - let date_matches = filters.from_date.map_or(true, |from| tx.block_timestamp >= from) - && filters.to_date.map_or(true, |to| tx.block_timestamp <= to); - if status_matches && date_matches { + if filters.is_status_match(&tx) && filters.is_date_match(&tx) { filtered_txs.push(tx); } } else { @@ -95,6 +90,21 @@ impl IndexedDbNftStorage { } } +impl NftTxHistoryFilters { + fn is_status_match(&self, tx: &NftTransferHistory) -> bool { + if !self.receive && !self.send { + true + } else { + (self.receive && tx.status == TransferStatus::Receive) || (self.send && tx.status == TransferStatus::Send) + } + } + + fn is_date_match(&self, tx: &NftTransferHistory) -> bool { + self.from_date.map_or(true, |from| tx.block_timestamp >= from) + && self.to_date.map_or(true, |to| tx.block_timestamp <= to) + } +} + #[async_trait] impl NftListStorageOps for IndexedDbNftStorage { type Error = WasmNftCacheError; From 23dd751a541138904c2d05ee8a9a412ee925786e Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 18 Jun 2023 13:19:15 +0700 Subject: [PATCH 087/112] move for_db_tests mod --- mm2src/coins/nft/nft_tests.rs | 498 +---------------------- mm2src/coins/nft/storage/for_db_tests.rs | 486 ++++++++++++++++++++++ mm2src/coins/nft/storage/mod.rs | 6 +- mm2src/coins/nft/storage/wasm/mod.rs | 4 +- 4 files changed, 494 insertions(+), 500 deletions(-) create mode 100644 mm2src/coins/nft/storage/for_db_tests.rs diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index f1f3a0bc77..a7f50c9e27 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -3,506 +3,12 @@ const NFT_HISTORY_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/0x const NFT_METADATA_URL_TEST: &str = "https://moralis-proxy.komodo.earth/api/v2/nft/0xed55e4477b795eaa9bb4bca24df42214e1a05c18/1111777?chain=POLYGON&format=decimal"; const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; -#[cfg(any(test, target_arch = "wasm32"))] -mod for_db_tests { - use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, - UriMeta}; - use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; - use mm2_number::BigDecimal; - use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; - use std::num::NonZeroUsize; - use std::str::FromStr; - - cfg_wasm32! { - use wasm_bindgen_test::*; - - wasm_bindgen_test_configure!(run_in_browser); - } - - const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; - const TOKEN_ID: &str = "214300044414"; - const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; - - fn nft() -> Nft { - Nft { - chain: Chain::Bsc, - token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), - token_id: Default::default(), - amount: BigDecimal::from_str("2").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "b34ddf294013d20a6d70691027625839".to_string(), - block_number_minted: 25465916, - block_number: 25919780, - contract_type: ContractType::Erc1155, - collection_name: None, - symbol: None, - token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), - metadata: Some("{\"name\":\"Tiki box\"}".to_string()), - last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), - last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), - minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: Some(false), - uri_meta: UriMeta { - image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), - token_name: None, - description: Some("Born to usher in Bull markets.".to_string()), - attributes: None, - animation_url: None, - }, - } - } - - fn nft_list() -> Vec { - let nft = Nft { - chain: Chain::Bsc, - token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), - token_id: Default::default(), - amount: BigDecimal::from_str("2").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "b34ddf294013d20a6d70691027625839".to_string(), - block_number_minted: 25465916, - block_number: 25919780, - contract_type: ContractType::Erc1155, - collection_name: None, - symbol: None, - token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), - metadata: Some("{\"name\":\"Tiki box\"}".to_string()), - last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), - last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), - minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: Some(false), - uri_meta: UriMeta { - image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), - token_name: None, - description: Some("Born to usher in Bull markets.".to_string()), - attributes: None, - animation_url: None, - }, - }; - - let nft1 = Nft { - chain: Chain::Bsc, - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300047252").unwrap(), - amount: BigDecimal::from_str("1").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "c5d1cfd75a0535b0ec750c0156e6ddfe".to_string(), - block_number_minted: 25721963, - block_number: 28056726, - contract_type: ContractType::Erc721, - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - symbol: Some("BMBBBF".to_string()), - token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), - metadata: Some( - "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" - .to_string(), - ), - last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), - last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), - minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: Some(false), - uri_meta: UriMeta { - image: Some( - "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), - ), - token_name: Some("Nebula Nodes".to_string()), - description: Some("Interchain nodes".to_string()), - attributes: None, - animation_url: None, - }, - }; - - let nft2 = Nft { - chain: Chain::Bsc, - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300044414").unwrap(), - amount: BigDecimal::from_str("1").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "125f8f4e952e107c257960000b4b250e".to_string(), - block_number_minted: 25810308, - block_number: 28056721, - contract_type: ContractType::Erc721, - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - symbol: Some("BMBBBF".to_string()), - token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300044414".to_string()), - metadata: Some( - "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" - .to_string(), - ), - last_token_uri_sync: Some("2023-02-19T19:12:09.471Z".to_string()), - last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), - minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: Some(false), - uri_meta: UriMeta { - image: Some( - "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), - ), - token_name: Some("Nebula Nodes".to_string()), - description: Some("Interchain nodes".to_string()), - attributes: None, - animation_url: None, - }, - }; - vec![nft, nft1, nft2] - } - - fn nft_tx_historty() -> Vec { - let tx = NftTransferHistory { - chain: Chain::Bsc, - block_number: 25919780, - block_timestamp: 1677166110, - block_hash: "0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string(), - transaction_hash: "0x9c16b962f63eead1c5d2355cc9037dde178b14b53043c57eb40c27964d22ae6a".to_string(), - transaction_index: 57, - log_index: 139, - value: Default::default(), - contract_type: ContractType::Erc1155, - transaction_type: "Single".to_string(), - token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), - token_id: Default::default(), - collection_name: None, - image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), - token_name: None, - from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: 1, - operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), - possible_spam: Some(false), - }; - - let tx1 = NftTransferHistory { - chain: Chain::Bsc, - block_number: 28056726, - block_timestamp: 1683627432, - block_hash: "0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string(), - transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), - transaction_index: 198, - log_index: 495, - value: Default::default(), - contract_type: ContractType::Erc721, - transaction_type: "Single".to_string(), - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300047252").unwrap(), - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), - token_name: Some("Nebula Nodes".to_string()), - from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: 1, - operator: None, - possible_spam: Some(false), - }; - - let tx2 = NftTransferHistory { - chain: Chain::Bsc, - block_number: 28056721, - block_timestamp: 1683627417, - block_hash: "0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string(), - transaction_hash: "0x981bad702cc6e088f0e9b5e7287ff7a3487b8d269103cee3b9e5803141f63f91".to_string(), - transaction_index: 83, - log_index: 201, - value: Default::default(), - contract_type: ContractType::Erc721, - transaction_type: "Single".to_string(), - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300044414").unwrap(), - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), - token_name: Some("Nebula Nodes".to_string()), - from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: 1, - operator: None, - possible_spam: Some(false), - }; - vec![tx, tx1, tx2] - } - - pub(crate) async fn test_add_get_nfts_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(nft.block_number, 28056721); - } - - pub(crate) async fn test_last_nft_blocks_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_scanned_block, last_block); - } - - pub(crate) async fn test_nft_list_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let nft_list = storage - .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 1); - let nft = nft_list.nfts.get(0).unwrap(); - assert_eq!(nft.block_number, 28056721); - assert_eq!(nft_list.skipped, 1); - assert_eq!(nft_list.total, 3); - } - - pub(crate) async fn test_remove_nft_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let remove_rslt = storage - .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) - .await - .unwrap(); - assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); - let list_len = storage - .get_nft_list(vec![chain], true, 1, None) - .await - .unwrap() - .nfts - .len(); - assert_eq!(list_len, 2); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 28056800); - } - - pub(crate) async fn test_nft_amount_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let mut nft = nft(); - storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - - nft.amount -= BigDecimal::from(1); - storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); - let amount = storage - .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "1"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919800); - - nft.amount += BigDecimal::from(1); - nft.block_number = 25919900; - storage - .update_nft_amount_and_block_number(&chain, nft.clone()) - .await - .unwrap(); - let amount = storage - .get_nft_amount(&chain, nft.token_address, nft.token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "2"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919900); - } - - pub(crate) async fn test_refresh_metadata_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - - let new_symbol = "NEW_SYMBOL"; - let mut nft = nft(); - storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - nft.symbol = Some(new_symbol.to_string()); - drop_mutability!(nft); - let token_add = nft.token_address.clone(); - let token_id = nft.token_id.clone(); - storage.refresh_nft_metadata(&chain, nft).await.unwrap(); - let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); - assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); - } - - pub(crate) async fn test_add_get_txs_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let tx1 = storage - .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .get(0) - .unwrap() - .clone(); - assert_eq!(tx1.block_number, 28056721); - let tx2 = storage - .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(tx2.block_number, 28056726); - let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); - assert_eq!(tx_from.len(), 2); - } - - pub(crate) async fn test_last_tx_block_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); - } - - pub(crate) async fn test_tx_history_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let tx_history = storage - .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) - .await - .unwrap(); - assert_eq!(tx_history.transfer_history.len(), 1); - let tx = tx_history.transfer_history.get(0).unwrap(); - assert_eq!(tx.block_number, 28056721); - assert_eq!(tx_history.skipped, 1); - assert_eq!(tx_history.total, 3); - } - - pub(crate) async fn test_tx_history_filters_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let filters = NftTxHistoryFilters { - receive: true, - send: false, - from_date: None, - to_date: None, - }; - - let filters1 = NftTxHistoryFilters { - receive: false, - send: false, - from_date: None, - to_date: Some(1677166110), - }; - - let filters2 = NftTxHistoryFilters { - receive: false, - send: false, - from_date: Some(1677166110), - to_date: Some(1683627417), - }; - - let tx_history = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(tx_history.transfer_history.len(), 3); - let tx = tx_history.transfer_history.get(0).unwrap(); - assert_eq!(tx.block_number, 28056726); - - let tx_history1 = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters1)) - .await - .unwrap(); - assert_eq!(tx_history1.transfer_history.len(), 1); - let tx1 = tx_history1.transfer_history.get(0).unwrap(); - assert_eq!(tx1.block_number, 25919780); - - let tx_history2 = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters2)) - .await - .unwrap(); - assert_eq!(tx_history2.transfer_history.len(), 2); - let tx_0 = tx_history2.transfer_history.get(0).unwrap(); - assert_eq!(tx_0.block_number, 28056721); - let tx_1 = tx_history2.transfer_history.get(1).unwrap(); - assert_eq!(tx_1.block_number, 25919780); - } -} - #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; - use crate::nft::nft_tests::for_db_tests::*; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::send_request_to_uri; + use crate::nft::storage::for_db_tests::*; use common::block_on; #[test] @@ -569,9 +75,9 @@ mod native_tests { #[cfg(target_arch = "wasm32")] mod wasm_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper}; - use crate::nft::nft_tests::for_db_tests::*; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::send_request_to_uri; + use crate::nft::storage::for_db_tests::*; use wasm_bindgen_test::*; wasm_bindgen_test_configure!(run_in_browser); diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs new file mode 100644 index 0000000000..59007fff03 --- /dev/null +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -0,0 +1,486 @@ +use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, + UriMeta}; +use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; +use mm2_number::BigDecimal; +use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; +use std::num::NonZeroUsize; +use std::str::FromStr; + +cfg_wasm32! { + use wasm_bindgen_test::*; + + wasm_bindgen_test_configure!(run_in_browser); +} + +const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; +const TOKEN_ID: &str = "214300044414"; +const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; + +fn nft() -> Nft { + Nft { + chain: Chain::Bsc, + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + amount: BigDecimal::from_str("2").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "b34ddf294013d20a6d70691027625839".to_string(), + block_number_minted: 25465916, + block_number: 25919780, + contract_type: ContractType::Erc1155, + collection_name: None, + symbol: None, + token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + metadata: Some("{\"name\":\"Tiki box\"}".to_string()), + last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), + last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), + minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + token_name: None, + description: Some("Born to usher in Bull markets.".to_string()), + attributes: None, + animation_url: None, + }, + } +} + +fn nft_list() -> Vec { + let nft = Nft { + chain: Chain::Bsc, + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + amount: BigDecimal::from_str("2").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "b34ddf294013d20a6d70691027625839".to_string(), + block_number_minted: 25465916, + block_number: 25919780, + contract_type: ContractType::Erc1155, + collection_name: None, + symbol: None, + token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + metadata: Some("{\"name\":\"Tiki box\"}".to_string()), + last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), + last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), + minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + token_name: None, + description: Some("Born to usher in Bull markets.".to_string()), + attributes: None, + animation_url: None, + }, + }; + + let nft1 = Nft { + chain: Chain::Bsc, + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "c5d1cfd75a0535b0ec750c0156e6ddfe".to_string(), + block_number_minted: 25721963, + block_number: 28056726, + contract_type: ContractType::Erc721, + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + symbol: Some("BMBBBF".to_string()), + token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), + metadata: Some( + "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), + last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), + minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + description: Some("Interchain nodes".to_string()), + attributes: None, + animation_url: None, + }, + }; + + let nft2 = Nft { + chain: Chain::Bsc, + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300044414").unwrap(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: "125f8f4e952e107c257960000b4b250e".to_string(), + block_number_minted: 25810308, + block_number: 28056721, + contract_type: ContractType::Erc721, + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + symbol: Some("BMBBBF".to_string()), + token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300044414".to_string()), + metadata: Some( + "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-19T19:12:09.471Z".to_string()), + last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), + minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), + possible_spam: Some(false), + uri_meta: UriMeta { + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + description: Some("Interchain nodes".to_string()), + attributes: None, + animation_url: None, + }, + }; + vec![nft, nft1, nft2] +} + +fn nft_tx_historty() -> Vec { + let tx = NftTransferHistory { + chain: Chain::Bsc, + block_number: 25919780, + block_timestamp: 1677166110, + block_hash: "0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string(), + transaction_hash: "0x9c16b962f63eead1c5d2355cc9037dde178b14b53043c57eb40c27964d22ae6a".to_string(), + transaction_index: 57, + log_index: 139, + value: Default::default(), + contract_type: ContractType::Erc1155, + transaction_type: "Single".to_string(), + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + collection_name: None, + image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + token_name: None, + from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), + possible_spam: Some(false), + }; + + let tx1 = NftTransferHistory { + chain: Chain::Bsc, + block_number: 28056726, + block_timestamp: 1683627432, + block_hash: "0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string(), + transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), + transaction_index: 198, + log_index: 495, + value: Default::default(), + contract_type: ContractType::Erc721, + transaction_type: "Single".to_string(), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: None, + possible_spam: Some(false), + }; + + let tx2 = NftTransferHistory { + chain: Chain::Bsc, + block_number: 28056721, + block_timestamp: 1683627417, + block_hash: "0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string(), + transaction_hash: "0x981bad702cc6e088f0e9b5e7287ff7a3487b8d269103cee3b9e5803141f63f91".to_string(), + transaction_index: 83, + log_index: 201, + value: Default::default(), + contract_type: ContractType::Erc721, + transaction_type: "Single".to_string(), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300044414").unwrap(), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: None, + possible_spam: Some(false), + }; + vec![tx, tx1, tx2] +} + +pub(crate) async fn test_add_get_nfts_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); +} + +pub(crate) async fn test_last_nft_blocks_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_scanned_block, last_block); +} + +pub(crate) async fn test_nft_list_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let nft_list = storage + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 1); + let nft = nft_list.nfts.get(0).unwrap(); + assert_eq!(nft.block_number, 28056721); + assert_eq!(nft_list.skipped, 1); + assert_eq!(nft_list.total, 3); +} + +pub(crate) async fn test_remove_nft_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let remove_rslt = storage + .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) + .await + .unwrap(); + assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); + let list_len = storage + .get_nft_list(vec![chain], true, 1, None) + .await + .unwrap() + .nfts + .len(); + assert_eq!(list_len, 2); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 28056800); +} + +pub(crate) async fn test_nft_amount_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + + nft.amount -= BigDecimal::from(1); + storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "1"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919800); + + nft.amount += BigDecimal::from(1); + nft.block_number = 25919900; + storage + .update_nft_amount_and_block_number(&chain, nft.clone()) + .await + .unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address, nft.token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "2"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919900); +} + +pub(crate) async fn test_refresh_metadata_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftListStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + + let new_symbol = "NEW_SYMBOL"; + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + nft.symbol = Some(new_symbol.to_string()); + drop_mutability!(nft); + let token_add = nft.token_address.clone(); + let token_id = nft.token_id.clone(); + storage.refresh_nft_metadata(&chain, nft).await.unwrap(); + let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); +} + +pub(crate) async fn test_add_get_txs_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let tx1 = storage + .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .get(0) + .unwrap() + .clone(); + assert_eq!(tx1.block_number, 28056721); + let tx2 = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(tx2.block_number, 28056726); + let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); + assert_eq!(tx_from.len(), 2); +} + +pub(crate) async fn test_last_tx_block_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); +} + +pub(crate) async fn test_tx_history_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let tx_history = storage + .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 1); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056721); + assert_eq!(tx_history.skipped, 1); + assert_eq!(tx_history.total, 3); +} + +pub(crate) async fn test_tx_history_filters_impl() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let filters = NftTxHistoryFilters { + receive: true, + send: false, + from_date: None, + to_date: None, + }; + + let filters1 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: None, + to_date: Some(1677166110), + }; + + let filters2 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: Some(1677166110), + to_date: Some(1683627417), + }; + + let tx_history = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 3); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056726); + + let tx_history1 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap(); + assert_eq!(tx_history1.transfer_history.len(), 1); + let tx1 = tx_history1.transfer_history.get(0).unwrap(); + assert_eq!(tx1.block_number, 25919780); + + let tx_history2 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters2)) + .await + .unwrap(); + assert_eq!(tx_history2.transfer_history.len(), 2); + let tx_0 = tx_history2.transfer_history.get(0).unwrap(); + assert_eq!(tx_0.block_number, 28056721); + let tx_1 = tx_history2.transfer_history.get(1).unwrap(); + assert_eq!(tx_1.block_number, 25919780); +} diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index 11af867f69..987475168b 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -10,8 +10,10 @@ use mm2_number::BigDecimal; use serde::{Deserialize, Serialize}; use std::num::NonZeroUsize; -#[cfg(not(target_arch = "wasm32"))] pub mod sql_storage; -#[cfg(target_arch = "wasm32")] pub mod wasm; +#[cfg(any(test, target_arch = "wasm32"))] +pub(crate) mod for_db_tests; +#[cfg(not(target_arch = "wasm32"))] pub(crate) mod sql_storage; +#[cfg(target_arch = "wasm32")] pub(crate) mod wasm; #[derive(Debug, PartialEq)] pub enum RemoveNftResult { diff --git a/mm2src/coins/nft/storage/wasm/mod.rs b/mm2src/coins/nft/storage/wasm/mod.rs index c7d5cd7a1e..6bbc8738c4 100644 --- a/mm2src/coins/nft/storage/wasm/mod.rs +++ b/mm2src/coins/nft/storage/wasm/mod.rs @@ -3,8 +3,8 @@ use mm2_db::indexed_db::{DbTransactionError, InitDbError}; use mm2_err_handle::prelude::*; use mm2_number::bigdecimal::ParseBigDecimalError; -pub mod nft_idb; -pub mod wasm_storage; +pub(crate) mod nft_idb; +pub(crate) mod wasm_storage; pub type WasmNftCacheResult = MmResult; From c7c424f7225edbb417dadcc37663daf916df3622 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 18 Jun 2023 14:36:31 +0700 Subject: [PATCH 088/112] test_get_update_tx_meta --- mm2src/coins/nft/nft_tests.rs | 6 ++ mm2src/coins/nft/storage/for_db_tests.rs | 84 ++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index a7f50c9e27..897729b7f0 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -70,6 +70,9 @@ mod native_tests { #[test] fn test_tx_history_filters() { block_on(test_tx_history_filters_impl()) } + + #[test] + fn test_get_update_tx() { block_on(test_get_update_tx_meta()) } } #[cfg(target_arch = "wasm32")] @@ -138,4 +141,7 @@ mod wasm_tests { #[wasm_bindgen_test] async fn test_tx_history_filters() { test_tx_history_filters_impl().await } + + #[wasm_bindgen_test] + async fn test_get_update_tx() { test_get_update_tx_meta().await } } diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs index 59007fff03..0ea53401e1 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -1,5 +1,5 @@ use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, - UriMeta}; + TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; @@ -45,6 +45,33 @@ fn nft() -> Nft { } } +fn tx() -> NftTransferHistory { + NftTransferHistory { + chain: Chain::Bsc, + block_number: 28056726, + block_timestamp: 1683627432, + block_hash: "0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string(), + transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), + transaction_index: 198, + log_index: 495, + value: Default::default(), + contract_type: ContractType::Erc721, + transaction_type: "Single".to_string(), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + token_name: Some("Nebula Nodes".to_string()), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, + amount: BigDecimal::from_str("1").unwrap(), + verified: 1, + operator: None, + possible_spam: Some(false), + } +} + fn nft_list() -> Vec { let nft = Nft { chain: Chain::Bsc, @@ -150,7 +177,7 @@ fn nft_tx_historty() -> Vec { token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), token_id: Default::default(), collection_name: None, - image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + image: None, token_name: None, from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), @@ -174,9 +201,9 @@ fn nft_tx_historty() -> Vec { transaction_type: "Single".to_string(), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300047252").unwrap(), - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), - token_name: Some("Nebula Nodes".to_string()), + collection_name: None, + image: None, + token_name: None, from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, @@ -484,3 +511,50 @@ pub(crate) async fn test_tx_history_filters_impl() { let tx_1 = tx_history2.transfer_history.get(1).unwrap(); assert_eq!(tx_1.block_number, 25919780); } + +pub(crate) async fn test_get_update_tx_meta() { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + let chain = Chain::Bsc; + NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); + assert!(is_initialized); + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); + assert_eq!(vec_token_add_id.len(), 2); + + let token_add_id = vec_token_add_id.get(0).unwrap(); + assert_eq!(BigDecimal::default(), token_add_id.token_id); + let tx_meta = TxMeta { + token_address: token_add_id.token_address.clone(), + token_id: token_add_id.token_id.clone(), + collection_name: None, + image: None, + token_name: Some("Tiki box".to_string()), + }; + storage.update_txs_meta_by_token_addr_id(&chain, tx_meta).await.unwrap(); + let tx_upd = storage + .get_txs_by_token_addr_id( + &chain, + token_add_id.token_address.clone(), + token_add_id.token_id.clone(), + ) + .await + .unwrap(); + let tx_upd = tx_upd.get(0).unwrap(); + assert_eq!(tx_upd.token_name, Some("Tiki box".to_string())); + + let token_add_id_1 = vec_token_add_id.get(1).unwrap(); + let token_id_1 = BigDecimal::from_str("214300047252").unwrap(); + assert_eq!(token_id_1, token_add_id_1.token_id); + + let tx_meta = tx(); + storage.update_tx_meta_by_hash(&chain, tx_meta).await.unwrap(); + let tx_by_hash = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(tx_by_hash.token_name, Some("Nebula Nodes".to_string())) +} From d2c1e5ee1c8c71609ebca1d8722bb82416e8a4f3 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 18 Jun 2023 15:13:35 +0700 Subject: [PATCH 089/112] check just len in test_get_update_tx_meta --- mm2src/coins/nft/nft_tests.rs | 4 ++-- mm2src/coins/nft/storage/for_db_tests.rs | 19 +++++-------------- 2 files changed, 7 insertions(+), 16 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 897729b7f0..31dece5ee9 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -72,7 +72,7 @@ mod native_tests { fn test_tx_history_filters() { block_on(test_tx_history_filters_impl()) } #[test] - fn test_get_update_tx() { block_on(test_get_update_tx_meta()) } + fn test_get_update_tx_meta() { block_on(test_get_update_tx_meta_impl()) } } #[cfg(target_arch = "wasm32")] @@ -143,5 +143,5 @@ mod wasm_tests { async fn test_tx_history_filters() { test_tx_history_filters_impl().await } #[wasm_bindgen_test] - async fn test_get_update_tx() { test_get_update_tx_meta().await } + async fn test_get_update_tx_meta() { test_get_update_tx_meta_impl().await } } diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs index 0ea53401e1..4adaad1f73 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -512,7 +512,7 @@ pub(crate) async fn test_tx_history_filters_impl() { assert_eq!(tx_1.block_number, 25919780); } -pub(crate) async fn test_get_update_tx_meta() { +pub(crate) async fn test_get_update_tx_meta_impl() { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; @@ -524,31 +524,22 @@ pub(crate) async fn test_get_update_tx_meta() { let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); assert_eq!(vec_token_add_id.len(), 2); - let token_add_id = vec_token_add_id.get(0).unwrap(); - assert_eq!(BigDecimal::default(), token_add_id.token_id); + let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); let tx_meta = TxMeta { - token_address: token_add_id.token_address.clone(), - token_id: token_add_id.token_id.clone(), + token_address: token_add.clone(), + token_id: Default::default(), collection_name: None, image: None, token_name: Some("Tiki box".to_string()), }; storage.update_txs_meta_by_token_addr_id(&chain, tx_meta).await.unwrap(); let tx_upd = storage - .get_txs_by_token_addr_id( - &chain, - token_add_id.token_address.clone(), - token_add_id.token_id.clone(), - ) + .get_txs_by_token_addr_id(&chain, token_add, Default::default()) .await .unwrap(); let tx_upd = tx_upd.get(0).unwrap(); assert_eq!(tx_upd.token_name, Some("Tiki box".to_string())); - let token_add_id_1 = vec_token_add_id.get(1).unwrap(); - let token_id_1 = BigDecimal::from_str("214300047252").unwrap(); - assert_eq!(token_id_1, token_add_id_1.token_id); - let tx_meta = tx(); storage.update_tx_meta_by_hash(&chain, tx_meta).await.unwrap(); let tx_by_hash = storage From c77d23677a2b6ef65bf0ec026b4c9bcf526f7517 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 19 Jun 2023 18:09:09 +0700 Subject: [PATCH 090/112] get_uri_meta, merge_from for UriMeta --- mm2src/coins/nft.rs | 40 +++++++++++++++--------- mm2src/coins/nft/nft_structs.rs | 36 ++++++++++++++++++++- mm2src/coins/nft/storage/for_db_tests.rs | 16 ++++++++++ 3 files changed, 76 insertions(+), 16 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 78e078c9d3..9f779d9bee 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -15,7 +15,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, N TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; +use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta, UriMetaFromStr}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use http::header::ACCEPT; @@ -156,7 +156,7 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu chain: req.chain, }; let mut nft_db = get_nft_metadata(ctx, req).await?; - let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; + let uri_meta = get_uri_meta(&moralis_meta.token_uri, &moralis_meta.metadata).await; nft_db.collection_name = moralis_meta.collection_name; nft_db.symbol = moralis_meta.symbol; nft_db.token_uri = moralis_meta.token_uri; @@ -209,7 +209,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult Some(contract_type) => contract_type.0, None => continue, }; - let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; + let uri_meta = get_uri_meta(&nft_wrapper.token_uri, &nft_wrapper.metadata).await; let nft = Nft { chain: *chain, token_address: nft_wrapper.token_address, @@ -360,7 +360,7 @@ async fn get_moralis_metadata( Some(contract_type) => contract_type.0, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let uri_meta = try_get_uri_meta(&nft_wrapper.token_uri).await?; + let uri_meta = get_uri_meta(&nft_wrapper.token_uri, &nft_wrapper.metadata).await; let nft_metadata = Nft { chain: *chain, token_address: nft_wrapper.token_address, @@ -445,18 +445,28 @@ async fn send_request_to_uri(uri: &str) -> MmResult { Ok(response) } -async fn try_get_uri_meta(token_uri: &Option) -> MmResult { - match token_uri { - Some(token_uri) => { - if let Ok(response_meta) = send_request_to_uri(token_uri).await { - let uri_meta_res: UriMeta = serde_json::from_str(&response_meta.to_string())?; - Ok(uri_meta_res) - } else { - Ok(UriMeta::default()) +async fn try_get_uri_meta(token_uri: &Option) -> UriMeta { + let mut uri_meta = UriMeta::default(); + if let Some(token_uri) = token_uri { + if let Ok(response_meta) = send_request_to_uri(token_uri).await { + if let Ok(uri_meta_res) = serde_json::from_str::(&response_meta.to_string()) { + uri_meta.merge_from(uri_meta_res); } - }, - None => Ok(UriMeta::default()), + } + } + drop_mutability!(uri_meta); + uri_meta +} + +async fn get_uri_meta(token_uri: &Option, metadata: &Option) -> UriMeta { + let mut uri_meta = try_get_uri_meta(token_uri).await; + if let Some(metadata) = metadata { + if let Ok(meta_from_meta) = serde_json::from_str::(metadata) { + uri_meta.merge_from(meta_from_meta); + } } + drop_mutability!(uri_meta); + uri_meta } fn get_tx_status(my_wallet: &str, to_address: &str) -> TransferStatus { @@ -643,7 +653,7 @@ async fn handle_receive_erc1155( storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } else { let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; - let uri_meta = try_get_uri_meta(&moralis_meta.token_uri).await?; + let uri_meta = get_uri_meta(&moralis_meta.token_uri, &moralis_meta.metadata).await; let nft = Nft { chain: *chain, token_address: moralis_meta.token_address, diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index c546a7ca79..a36b7dcc8c 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -129,13 +129,47 @@ impl fmt::Display for ContractType { } #[derive(Clone, Debug, Default, Deserialize, Serialize)] -pub(crate) struct UriMeta { +pub(crate) struct UriMetaFromStr { pub(crate) image: Option, + /// the same as image, some tokens use **image_url** name for **image** field + /// but sometimes token_uri contains both`image_url` and `image` + pub(crate) image_url: Option, #[serde(rename(deserialize = "name"))] pub(crate) token_name: Option, pub(crate) description: Option, pub(crate) attributes: Option, pub(crate) animation_url: Option, + pub(crate) external_url: Option, + pub(crate) image_details: Option, + pub(crate) dna: Option, + pub(crate) compiler: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +pub(crate) struct UriMeta { + pub(crate) image: Option, + pub(crate) token_name: Option, + pub(crate) description: Option, + pub(crate) attributes: Option, + pub(crate) animation_url: Option, + pub(crate) external_url: Option, + pub(crate) image_details: Option, + pub(crate) dna: Option, + pub(crate) compiler: Option, +} + +impl UriMeta { + pub(crate) fn merge_from(&mut self, other: UriMetaFromStr) { + self.image = self.image.clone().or(other.image).or(other.image_url); + self.token_name = self.token_name.clone().or(other.token_name); + self.description = self.description.clone().or(other.description); + self.attributes = self.attributes.clone().or(other.attributes); + self.animation_url = self.animation_url.clone().or(other.animation_url); + self.external_url = self.external_url.clone().or(other.external_url); + self.image_details = self.image_details.clone().or(other.image_details); + self.dna = self.dna.clone().or(other.dna); + self.compiler = self.compiler.clone().or(other.compiler); + } } #[derive(Clone, Debug, Deserialize, Serialize)] diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs index 4adaad1f73..da3b448845 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -41,6 +41,10 @@ fn nft() -> Nft { description: Some("Born to usher in Bull markets.".to_string()), attributes: None, animation_url: None, + external_url: None, + image_details: None, + dna: None, + compiler: None, }, } } @@ -97,6 +101,10 @@ fn nft_list() -> Vec { description: Some("Born to usher in Bull markets.".to_string()), attributes: None, animation_url: None, + external_url: None, + image_details: None, + dna: None, + compiler: None, }, }; @@ -127,6 +135,10 @@ fn nft_list() -> Vec { description: Some("Interchain nodes".to_string()), attributes: None, animation_url: None, + external_url: None, + image_details: None, + dna: None, + compiler: None, }, }; @@ -157,6 +169,10 @@ fn nft_list() -> Vec { description: Some("Interchain nodes".to_string()), attributes: None, animation_url: None, + external_url: None, + image_details: None, + dna: None, + compiler: None, }, }; vec![nft, nft1, nft2] From 5992d28914908e1a544dd48de67b7bef40adaf47 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 20 Jun 2023 10:27:45 +0700 Subject: [PATCH 091/112] add token_uri in txs, json in image_details --- mm2src/coins/nft.rs | 10 ++++++++++ mm2src/coins/nft/nft_structs.rs | 6 ++++-- mm2src/coins/nft/storage/for_db_tests.rs | 5 +++++ mm2src/coins/nft/storage/sql_storage.rs | 6 +++++- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 9 ++++++++- 5 files changed, 32 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 9f779d9bee..ac834d4ce5 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -172,6 +172,7 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu let tx_meta = TxMeta { token_address: nft_db.token_address, token_id: nft_db.token_id, + token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, image: nft_db.uri_meta.image, token_name: nft_db.uri_meta.token_name, @@ -305,6 +306,7 @@ async fn get_moralis_nft_transfers( transaction_type: transfer_wrapper.transaction_type, token_address: transfer_wrapper.token_address, token_id: transfer_wrapper.token_id.0, + token_uri: None, collection_name: None, image: None, token_name: None, @@ -532,6 +534,7 @@ async fn handle_send_erc721( let tx_meta = TxMeta { token_address: nft_db.token_address, token_id: nft_db.token_id, + token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, image: nft_db.uri_meta.image, token_name: nft_db.uri_meta.token_name, @@ -562,6 +565,7 @@ async fn handle_receive_erc721( let tx_meta = TxMeta { token_address: nft.token_address, token_id: nft.token_id, + token_uri: nft.token_uri, collection_name: nft.collection_name, image: nft.uri_meta.image, token_name: nft.uri_meta.token_name, @@ -587,6 +591,7 @@ async fn handle_send_erc1155( let tx_meta = TxMeta { token_address: nft_db.token_address, token_id: nft_db.token_id, + token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, image: nft_db.uri_meta.image, token_name: nft_db.uri_meta.token_name, @@ -605,6 +610,7 @@ async fn handle_send_erc1155( let tx_meta = TxMeta { token_address: nft_db.token_address, token_id: nft_db.token_id, + token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, image: nft_db.uri_meta.image, token_name: nft_db.uri_meta.token_name, @@ -646,6 +652,7 @@ async fn handle_receive_erc1155( let tx_meta = TxMeta { token_address: nft_db.token_address, token_id: nft_db.token_id, + token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, image: nft_db.uri_meta.image, token_name: nft_db.uri_meta.token_name, @@ -678,6 +685,7 @@ async fn handle_receive_erc1155( let tx_meta = TxMeta { token_address: nft.token_address, token_id: nft.token_id, + token_uri: nft.token_uri, collection_name: nft.collection_name, image: nft.uri_meta.image, token_name: nft.uri_meta.token_name, @@ -734,6 +742,7 @@ where let tx_meta = TxMeta { token_address: nft.token_address, token_id: nft.token_id, + token_uri: nft.token_uri, collection_name: nft.collection_name, image: nft.uri_meta.image, token_name: nft.uri_meta.token_name, @@ -754,6 +763,7 @@ where let tx_meta = TxMeta { token_address: nft_meta.token_address, token_id: nft_meta.token_id, + token_uri: nft_meta.token_uri, collection_name: nft_meta.collection_name, image: nft_meta.uri_meta.image, token_name: nft_meta.uri_meta.token_name, diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index a36b7dcc8c..5672e6dd1d 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -140,7 +140,7 @@ pub(crate) struct UriMetaFromStr { pub(crate) attributes: Option, pub(crate) animation_url: Option, pub(crate) external_url: Option, - pub(crate) image_details: Option, + pub(crate) image_details: Option, pub(crate) dna: Option, pub(crate) compiler: Option, } @@ -153,7 +153,7 @@ pub(crate) struct UriMeta { pub(crate) attributes: Option, pub(crate) animation_url: Option, pub(crate) external_url: Option, - pub(crate) image_details: Option, + pub(crate) image_details: Option, pub(crate) dna: Option, pub(crate) compiler: Option, } @@ -361,6 +361,7 @@ pub struct NftTransferHistory { pub(crate) transaction_type: String, pub(crate) token_address: String, pub(crate) token_id: BigDecimal, + pub(crate) token_uri: Option, pub(crate) collection_name: Option, pub(crate) image: Option, pub(crate) token_name: Option, @@ -428,6 +429,7 @@ pub struct NftTokenAddrId { pub struct TxMeta { pub(crate) token_address: String, pub(crate) token_id: BigDecimal, + pub(crate) token_uri: Option, pub(crate) collection_name: Option, pub(crate) image: Option, pub(crate) token_name: Option, diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs index da3b448845..3c8bc3c073 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -63,6 +63,7 @@ fn tx() -> NftTransferHistory { transaction_type: "Single".to_string(), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300047252").unwrap(), + token_uri: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), token_name: Some("Nebula Nodes".to_string()), @@ -192,6 +193,7 @@ fn nft_tx_historty() -> Vec { transaction_type: "Single".to_string(), token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), token_id: Default::default(), + token_uri: None, collection_name: None, image: None, token_name: None, @@ -217,6 +219,7 @@ fn nft_tx_historty() -> Vec { transaction_type: "Single".to_string(), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300047252").unwrap(), + token_uri: None, collection_name: None, image: None, token_name: None, @@ -242,6 +245,7 @@ fn nft_tx_historty() -> Vec { transaction_type: "Single".to_string(), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300044414").unwrap(), + token_uri: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), token_name: Some("Nebula Nodes".to_string()), @@ -544,6 +548,7 @@ pub(crate) async fn test_get_update_tx_meta_impl() { let tx_meta = TxMeta { token_address: token_add.clone(), token_id: Default::default(), + token_uri: None, collection_name: None, image: None, token_name: Some("Tiki box".to_string()), diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 4fed37e5f4..b7b0faadae 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -58,6 +58,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { token_id VARCHAR(256) NOT NULL, status TEXT NOT NULL, amount VARCHAR(256) NOT NULL, + token_uri TEXT, collection_name TEXT, image TEXT, token_name TEXT, @@ -274,7 +275,7 @@ fn update_meta_by_tx_hash_sql(chain: &Chain) -> MmResult { validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET collection_name = ?1, image = ?2, token_name = ?3, details_json = ?4 WHERE transaction_hash = ?5;", + "UPDATE {} SET token_uri = ?1, collection_name = ?2, image = ?3, token_name = ?4, details_json = ?5 WHERE transaction_hash = ?6;", table_name ); Ok(sql) @@ -409,6 +410,7 @@ fn get_txs_with_empty_meta_builder<'a>(conn: &'a Connection, chain: &'a Chain) - .distinct() .field("token_address") .field("token_id") + .and_where_is_null("token_uri") .and_where_is_null("collection_name") .and_where_is_null("image") .and_where_is_null("token_name"); @@ -839,6 +841,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let sql = update_meta_by_tx_hash_sql(chain)?; let tx_json = json::to_string(&tx).expect("serialization should not fail"); let params = [ + tx.token_uri, tx.collection_name, tx.image, tx.token_name, @@ -862,6 +865,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { .get_txs_by_token_addr_id(chain, tx_meta.token_address, tx_meta.token_id) .await?; for mut tx in txs.into_iter() { + tx.token_uri = tx_meta.token_uri.clone(); tx.collection_name = tx_meta.collection_name.clone(); tx.image = tx_meta.image.clone(); tx.token_name = tx_meta.token_name.clone(); diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index e93d9174cc..885e1c4279 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -496,6 +496,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; for mut tx in txs { + tx.token_uri = tx_meta.token_uri.clone(); tx.collection_name = tx_meta.collection_name.clone(); tx.image = tx_meta.image.clone(); tx.token_name = tx_meta.token_name.clone(); @@ -527,7 +528,11 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? { - if item.collection_name.is_none() && item.image.is_none() && item.token_name.is_none() { + if item.token_uri.is_none() + && item.collection_name.is_none() + && item.image.is_none() + && item.token_name.is_none() + { res.insert(NftTokenAddrId { token_address: item.token_address, token_id: BigDecimal::from_str(&item.token_id).map_err(WasmNftCacheError::ParseBigDecimalError)?, @@ -599,6 +604,7 @@ pub(crate) struct NftTxHistoryTable { token_id: String, status: TransferStatus, amount: String, + token_uri: Option, collection_name: Option, image: Option, token_name: Option, @@ -624,6 +630,7 @@ impl NftTxHistoryTable { token_id: tx.token_id.to_string(), status: tx.status, amount: tx.amount.to_string(), + token_uri: tx.token_uri.clone(), collection_name: tx.collection_name.clone(), image: tx.image.clone(), token_name: tx.token_name.clone(), From 740a284992e8a687fed61a51be7745058b3233b5 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 21 Jun 2023 14:22:28 +0700 Subject: [PATCH 092/112] add check_moralis_ipfs_bafy and test it --- mm2src/coins/nft.rs | 37 +++++++++++++++++++++++++---------- mm2src/coins/nft/nft_tests.rs | 11 ++++++++++- 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index ac834d4ce5..32d9971229 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -156,10 +156,11 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu chain: req.chain, }; let mut nft_db = get_nft_metadata(ctx, req).await?; - let uri_meta = get_uri_meta(&moralis_meta.token_uri, &moralis_meta.metadata).await; + let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.metadata.as_deref()).await; nft_db.collection_name = moralis_meta.collection_name; nft_db.symbol = moralis_meta.symbol; - nft_db.token_uri = moralis_meta.token_uri; + nft_db.token_uri = token_uri; nft_db.metadata = moralis_meta.metadata; nft_db.last_token_uri_sync = moralis_meta.last_token_uri_sync; nft_db.last_metadata_sync = moralis_meta.last_metadata_sync; @@ -210,7 +211,8 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult Some(contract_type) => contract_type.0, None => continue, }; - let uri_meta = get_uri_meta(&nft_wrapper.token_uri, &nft_wrapper.metadata).await; + let token_uri = check_moralis_ipfs_bafy(nft_wrapper.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), nft_wrapper.metadata.as_deref()).await; let nft = Nft { chain: *chain, token_address: nft_wrapper.token_address, @@ -223,7 +225,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult contract_type, collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, + token_uri, metadata: nft_wrapper.metadata, last_token_uri_sync: nft_wrapper.last_token_uri_sync, last_metadata_sync: nft_wrapper.last_metadata_sync, @@ -362,7 +364,8 @@ async fn get_moralis_metadata( Some(contract_type) => contract_type.0, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let uri_meta = get_uri_meta(&nft_wrapper.token_uri, &nft_wrapper.metadata).await; + let token_uri = check_moralis_ipfs_bafy(nft_wrapper.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), nft_wrapper.metadata.as_deref()).await; let nft_metadata = Nft { chain: *chain, token_address: nft_wrapper.token_address, @@ -375,7 +378,7 @@ async fn get_moralis_metadata( contract_type, collection_name: nft_wrapper.name, symbol: nft_wrapper.symbol, - token_uri: nft_wrapper.token_uri, + token_uri, metadata: nft_wrapper.metadata, last_token_uri_sync: nft_wrapper.last_token_uri_sync, last_metadata_sync: nft_wrapper.last_metadata_sync, @@ -447,7 +450,18 @@ async fn send_request_to_uri(uri: &str) -> MmResult { Ok(response) } -async fn try_get_uri_meta(token_uri: &Option) -> UriMeta { +fn check_moralis_ipfs_bafy(token_uri: Option<&str>) -> Option { + token_uri.map(|uri| { + if uri.contains("https://ipfs.moralis.io") && uri.contains("bafy") { + let parts: Vec<&str> = uri.splitn(2, "/ipfs/").collect(); + format!("https://ipfs.io/ipfs/{}", parts[1]) + } else { + uri.to_string() + } + }) +} + +async fn try_get_uri_meta(token_uri: Option<&str>) -> UriMeta { let mut uri_meta = UriMeta::default(); if let Some(token_uri) = token_uri { if let Ok(response_meta) = send_request_to_uri(token_uri).await { @@ -460,13 +474,15 @@ async fn try_get_uri_meta(token_uri: &Option) -> UriMeta { uri_meta } -async fn get_uri_meta(token_uri: &Option, metadata: &Option) -> UriMeta { +async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>) -> UriMeta { let mut uri_meta = try_get_uri_meta(token_uri).await; if let Some(metadata) = metadata { if let Ok(meta_from_meta) = serde_json::from_str::(metadata) { uri_meta.merge_from(meta_from_meta); } } + uri_meta.image = check_moralis_ipfs_bafy(uri_meta.image.as_deref()); + uri_meta.animation_url = check_moralis_ipfs_bafy(uri_meta.animation_url.as_deref()); drop_mutability!(uri_meta); uri_meta } @@ -660,7 +676,8 @@ async fn handle_receive_erc1155( storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } else { let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; - let uri_meta = get_uri_meta(&moralis_meta.token_uri, &moralis_meta.metadata).await; + let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.metadata.as_deref()).await; let nft = Nft { chain: *chain, token_address: moralis_meta.token_address, @@ -673,7 +690,7 @@ async fn handle_receive_erc1155( contract_type: moralis_meta.contract_type, collection_name: moralis_meta.collection_name, symbol: moralis_meta.symbol, - token_uri: moralis_meta.token_uri, + token_uri, metadata: moralis_meta.metadata, last_token_uri_sync: moralis_meta.last_token_uri_sync, last_metadata_sync: moralis_meta.last_metadata_sync, diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 31dece5ee9..7ac4f66394 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -7,10 +7,19 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; mod native_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::send_request_to_uri; use crate::nft::storage::for_db_tests::*; + use crate::nft::{check_moralis_ipfs_bafy, send_request_to_uri}; use common::block_on; + #[test] + fn test_moralis_ipfs_bafy() { + let uri = + "https://ipfs.moralis.io:2053/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; + let res_uri = check_moralis_ipfs_bafy(Some(uri)); + let expected = "https://ipfs.io/ipfs/bafybeifnek24coy5xj5qabdwh24dlp5omq34nzgvazkfyxgnqms4eidsiq/1.json"; + assert_eq!(expected, res_uri.unwrap()); + } + #[test] fn test_moralis_nft_list() { let response = block_on(send_request_to_uri(NFT_LIST_URL_TEST)).unwrap(); From f2be86ab35384d34eed520a625563645c244c81e Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 22 Jun 2023 15:46:04 +0700 Subject: [PATCH 093/112] reduce code repetition in for_db_tests.rs --- mm2src/coins/nft/storage/for_db_tests.rs | 85 +++++++++--------------- 1 file changed, 30 insertions(+), 55 deletions(-) diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs index 3c8bc3c073..a5ac40e696 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -260,13 +260,27 @@ fn nft_tx_historty() -> Vec { vec![tx, tx1, tx2] } -pub(crate) async fn test_add_get_nfts_impl() { +async fn init_nft_list_storage(chain: &Chain) -> impl NftListStorageOps + NftTxHistoryStorageOps { let ctx = mm_ctx_with_custom_db(); let storage = NftStorageBuilder::new(&ctx).build().unwrap(); - let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); + NftListStorageOps::init(&storage, chain).await.unwrap(); + let is_initialized = NftListStorageOps::is_initialized(&storage, chain).await.unwrap(); + assert!(is_initialized); + storage +} + +async fn init_nft_history_storage(chain: &Chain) -> impl NftListStorageOps + NftTxHistoryStorageOps { + let ctx = mm_ctx_with_custom_db(); + let storage = NftStorageBuilder::new(&ctx).build().unwrap(); + NftTxHistoryStorageOps::init(&storage, chain).await.unwrap(); + let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, chain).await.unwrap(); assert!(is_initialized); + storage +} + +pub(crate) async fn test_add_get_nfts_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); @@ -280,12 +294,8 @@ pub(crate) async fn test_add_get_nfts_impl() { } pub(crate) async fn test_last_nft_blocks_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); @@ -298,12 +308,8 @@ pub(crate) async fn test_last_nft_blocks_impl() { } pub(crate) async fn test_nft_list_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); @@ -319,12 +325,8 @@ pub(crate) async fn test_nft_list_impl() { } pub(crate) async fn test_remove_nft_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); @@ -346,12 +348,8 @@ pub(crate) async fn test_remove_nft_impl() { } pub(crate) async fn test_nft_amount_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_list_storage(&chain).await; let mut nft = nft(); storage .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) @@ -386,13 +384,8 @@ pub(crate) async fn test_nft_amount_impl() { } pub(crate) async fn test_refresh_metadata_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftListStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftListStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); - + let storage = init_nft_list_storage(&chain).await; let new_symbol = "NEW_SYMBOL"; let mut nft = nft(); storage @@ -409,14 +402,11 @@ pub(crate) async fn test_refresh_metadata_impl() { } pub(crate) async fn test_add_get_txs_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); storage.add_txs_to_history(&chain, txs).await.unwrap(); + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); let tx1 = storage .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) @@ -437,12 +427,8 @@ pub(crate) async fn test_add_get_txs_impl() { } pub(crate) async fn test_last_tx_block_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); storage.add_txs_to_history(&chain, txs).await.unwrap(); @@ -454,12 +440,8 @@ pub(crate) async fn test_last_tx_block_impl() { } pub(crate) async fn test_tx_history_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); storage.add_txs_to_history(&chain, txs).await.unwrap(); @@ -475,12 +457,8 @@ pub(crate) async fn test_tx_history_impl() { } pub(crate) async fn test_tx_history_filters_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); storage.add_txs_to_history(&chain, txs).await.unwrap(); @@ -533,14 +511,11 @@ pub(crate) async fn test_tx_history_filters_impl() { } pub(crate) async fn test_get_update_tx_meta_impl() { - let ctx = mm_ctx_with_custom_db(); - let storage = NftStorageBuilder::new(&ctx).build().unwrap(); let chain = Chain::Bsc; - NftTxHistoryStorageOps::init(&storage, &chain).await.unwrap(); - let is_initialized = NftTxHistoryStorageOps::is_initialized(&storage, &chain).await.unwrap(); - assert!(is_initialized); + let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); storage.add_txs_to_history(&chain, txs).await.unwrap(); + let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); assert_eq!(vec_token_add_id.len(), 2); From 92f28aa033a303bdc53d662063d1e86a71310f24 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 23 Jun 2023 17:52:48 +0700 Subject: [PATCH 094/112] use protect_from_history_spam, protect_from_nft_spam --- mm2src/coins/nft.rs | 91 +++++++++++++++++++++++- mm2src/coins/nft/nft_structs.rs | 13 ++-- mm2src/coins/nft/storage/for_db_tests.rs | 8 --- 3 files changed, 95 insertions(+), 17 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 13929d2a60..2d9bc861d4 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -44,9 +44,15 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult MmResult MmResu token_address: req.token_address, token_id: req.token_id, chain: req.chain, + protect_from_spam: false, }; let mut nft_db = get_nft_metadata(ctx, req).await?; let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); @@ -788,3 +804,72 @@ where } Ok(()) } + +fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { + let mut is_spam = false; + if let Some(name) = &nft.collection_name { + if name.contains("http://") || name.contains("https://") { + nft.collection_name = Some("URL redacted for user protection".to_string()); + is_spam = true; + } + } + if let Some(symbol) = &nft.symbol { + if symbol.contains("http://") || symbol.contains("https://") { + nft.symbol = Some("URL redacted for user protection".to_string()); + is_spam = true; + } + } + if let Some(token_name) = &nft.uri_meta.token_name { + if token_name.contains("http://") || token_name.contains("https://") { + nft.uri_meta.token_name = Some("URL redacted for user protection".to_string()); + is_spam = true; + } + } + + if let Some(metadata_str) = &nft.metadata { + if let Ok(mut metadata) = serde_json::from_str::>(metadata_str) { + if let Some(name) = metadata.get("name").and_then(|v| v.as_str()) { + if name.contains("http://") || name.contains("https://") { + metadata.insert( + "name".to_string(), + serde_json::Value::String("URL redacted for user protection".to_string()), + ); + is_spam = true; + } + } + if let Some(symbol) = metadata.get("symbol").and_then(|v| v.as_str()) { + if symbol.contains("http://") || symbol.contains("https://") { + metadata.insert( + "symbol".to_string(), + serde_json::Value::String("URL redacted for user protection".to_string()), + ); + is_spam = true; + } + } + nft.metadata = Some(serde_json::to_string(&metadata)?); + } + } + if is_spam { + nft.possible_spam = Some(true); + } + Ok(()) +} + +fn protect_from_history_spam(tx: &mut NftTransferHistory) { + let mut is_spam = false; + if let Some(name) = &tx.collection_name { + if name.contains("http://") || name.contains("https://") { + tx.collection_name = Some("URL redacted for user protection".to_string()); + is_spam = true; + } + } + if let Some(token_name) = &tx.token_name { + if token_name.contains("http://") || token_name.contains("https://") { + tx.token_name = Some("URL redacted for user protection".to_string()); + is_spam = true; + } + } + if is_spam { + tx.possible_spam = Some(true); + } +} diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 5672e6dd1d..1cfc42ddd6 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -18,6 +18,8 @@ pub struct NftListReq { #[serde(default = "ten")] pub(crate) limit: usize, pub(crate) page_number: Option, + #[serde(default)] + pub(crate) protect_from_spam: bool, } #[derive(Debug, Deserialize)] @@ -25,6 +27,8 @@ pub struct NftMetadataReq { pub(crate) token_address: Address, pub(crate) token_id: BigDecimal, pub(crate) chain: Chain, + #[serde(default)] + pub(crate) protect_from_spam: bool, } #[derive(Debug, Deserialize)] @@ -141,8 +145,6 @@ pub(crate) struct UriMetaFromStr { pub(crate) animation_url: Option, pub(crate) external_url: Option, pub(crate) image_details: Option, - pub(crate) dna: Option, - pub(crate) compiler: Option, } #[derive(Clone, Debug, Default, Deserialize, Serialize)] @@ -154,8 +156,6 @@ pub(crate) struct UriMeta { pub(crate) animation_url: Option, pub(crate) external_url: Option, pub(crate) image_details: Option, - pub(crate) dna: Option, - pub(crate) compiler: Option, } impl UriMeta { @@ -167,8 +167,6 @@ impl UriMeta { self.animation_url = self.animation_url.clone().or(other.animation_url); self.external_url = self.external_url.clone().or(other.external_url); self.image_details = self.image_details.clone().or(other.image_details); - self.dna = self.dna.clone().or(other.dna); - self.compiler = self.compiler.clone().or(other.compiler); } } @@ -302,6 +300,7 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } +#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, @@ -311,6 +310,8 @@ pub struct NftTransfersReq { #[serde(default = "ten")] pub(crate) limit: usize, pub(crate) page_number: Option, + #[serde(default)] + pub(crate) protect_from_spam: bool, } #[derive(Debug, Display)] diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/for_db_tests.rs index a5ac40e696..0b036429cb 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/for_db_tests.rs @@ -43,8 +43,6 @@ fn nft() -> Nft { animation_url: None, external_url: None, image_details: None, - dna: None, - compiler: None, }, } } @@ -104,8 +102,6 @@ fn nft_list() -> Vec { animation_url: None, external_url: None, image_details: None, - dna: None, - compiler: None, }, }; @@ -138,8 +134,6 @@ fn nft_list() -> Vec { animation_url: None, external_url: None, image_details: None, - dna: None, - compiler: None, }, }; @@ -172,8 +166,6 @@ fn nft_list() -> Vec { animation_url: None, external_url: None, image_details: None, - dna: None, - compiler: None, }, }; vec![nft, nft1, nft2] From cc6ddd6ad9d8180210a32711d09869608ba220d1 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 23 Jun 2023 19:25:38 +0700 Subject: [PATCH 095/112] move logic spam in meta into separate function --- mm2src/coins/nft.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 2d9bc861d4..5df999b933 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -825,7 +825,15 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { is_spam = true; } } + let meta_spam = check_nft_metadata_for_spam(nft)?; + if is_spam || meta_spam { + nft.possible_spam = Some(true); + } + Ok(()) +} +fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { + let mut is_spam = false; if let Some(metadata_str) = &nft.metadata { if let Ok(mut metadata) = serde_json::from_str::>(metadata_str) { if let Some(name) = metadata.get("name").and_then(|v| v.as_str()) { @@ -834,6 +842,7 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { "name".to_string(), serde_json::Value::String("URL redacted for user protection".to_string()), ); + nft.metadata = Some(serde_json::to_string(&metadata)?); is_spam = true; } } @@ -843,16 +852,13 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { "symbol".to_string(), serde_json::Value::String("URL redacted for user protection".to_string()), ); + nft.metadata = Some(serde_json::to_string(&metadata)?); is_spam = true; } } - nft.metadata = Some(serde_json::to_string(&metadata)?); } } - if is_spam { - nft.possible_spam = Some(true); - } - Ok(()) + Ok(is_spam) } fn protect_from_history_spam(tx: &mut NftTransferHistory) { From 7d4c7fca7fdcc780c64ae99eff465569350af2f3 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 25 Jun 2023 18:09:21 +0700 Subject: [PATCH 096/112] reduce code duplication in check if spam --- mm2src/coins/nft.rs | 96 +++++++++++++++++++-------------------------- 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 5df999b933..ab73099687 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -805,77 +805,61 @@ where Ok(()) } -fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { - let mut is_spam = false; - if let Some(name) = &nft.collection_name { - if name.contains("http://") || name.contains("https://") { - nft.collection_name = Some("URL redacted for user protection".to_string()); - is_spam = true; - } - } - if let Some(symbol) = &nft.symbol { - if symbol.contains("http://") || symbol.contains("https://") { - nft.symbol = Some("URL redacted for user protection".to_string()); - is_spam = true; - } +fn check_and_redact_if_spam(value: &mut Option) -> bool { + match value { + Some(s) if s.contains("http://") || s.contains("https://") => { + *value = Some("URL redacted for user protection".to_string()); + true + }, + _ => false, } - if let Some(token_name) = &nft.uri_meta.token_name { - if token_name.contains("http://") || token_name.contains("https://") { - nft.uri_meta.token_name = Some("URL redacted for user protection".to_string()); - is_spam = true; - } +} + +fn protect_from_history_spam(tx: &mut NftTransferHistory) { + let collection_name_spam = check_and_redact_if_spam(&mut tx.collection_name); + let token_name_spam = check_and_redact_if_spam(&mut tx.token_name); + + if collection_name_spam || token_name_spam { + tx.possible_spam = Some(true); } +} + +fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { + let collection_name_spam = check_and_redact_if_spam(&mut nft.collection_name); + let symbol_spam = check_and_redact_if_spam(&mut nft.symbol); + let token_name_spam = check_and_redact_if_spam(&mut nft.uri_meta.token_name); let meta_spam = check_nft_metadata_for_spam(nft)?; - if is_spam || meta_spam { + + if collection_name_spam || symbol_spam || token_name_spam || meta_spam { nft.possible_spam = Some(true); } Ok(()) } fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { - let mut is_spam = false; if let Some(metadata_str) = &nft.metadata { if let Ok(mut metadata) = serde_json::from_str::>(metadata_str) { - if let Some(name) = metadata.get("name").and_then(|v| v.as_str()) { - if name.contains("http://") || name.contains("https://") { - metadata.insert( - "name".to_string(), - serde_json::Value::String("URL redacted for user protection".to_string()), - ); - nft.metadata = Some(serde_json::to_string(&metadata)?); - is_spam = true; - } - } - if let Some(symbol) = metadata.get("symbol").and_then(|v| v.as_str()) { - if symbol.contains("http://") || symbol.contains("https://") { - metadata.insert( - "symbol".to_string(), - serde_json::Value::String("URL redacted for user protection".to_string()), - ); - nft.metadata = Some(serde_json::to_string(&metadata)?); - is_spam = true; - } + if check_and_redact_metadata_field(&mut metadata, "name")? { + nft.metadata = Some(serde_json::to_string(&metadata)?); + return Ok(true); } } } - Ok(is_spam) + Ok(false) } -fn protect_from_history_spam(tx: &mut NftTransferHistory) { - let mut is_spam = false; - if let Some(name) = &tx.collection_name { - if name.contains("http://") || name.contains("https://") { - tx.collection_name = Some("URL redacted for user protection".to_string()); - is_spam = true; - } - } - if let Some(token_name) = &tx.token_name { - if token_name.contains("http://") || token_name.contains("https://") { - tx.token_name = Some("URL redacted for user protection".to_string()); - is_spam = true; - } - } - if is_spam { - tx.possible_spam = Some(true); +fn check_and_redact_metadata_field( + metadata: &mut serde_json::Map, + field: &str, +) -> MmResult { + match metadata.get(field).and_then(|v| v.as_str()) { + Some(value) if value.contains("http://") || value.contains("https://") => { + metadata.insert( + field.to_string(), + serde_json::Value::String("URL redacted for user protection".to_string()), + ); + Ok(true) + }, + _ => Ok(false), } } From a17e346224fa4d9a9557403c3af0164e3b35f9fb Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 25 Jun 2023 19:42:59 +0700 Subject: [PATCH 097/112] remove dead_code annotation --- mm2src/coins/nft/nft_structs.rs | 1 - mm2src/coins/nft/storage/wasm/nft_idb.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 1cfc42ddd6..bddbe6e855 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -300,7 +300,6 @@ pub struct TransactionNftDetails { pub(crate) transaction_type: TransactionType, } -#[allow(dead_code)] #[derive(Debug, Deserialize)] pub struct NftTransfersReq { pub(crate) chains: Vec, diff --git a/mm2src/coins/nft/storage/wasm/nft_idb.rs b/mm2src/coins/nft/storage/wasm/nft_idb.rs index 4f752ef277..287bcab30a 100644 --- a/mm2src/coins/nft/storage/wasm/nft_idb.rs +++ b/mm2src/coins/nft/storage/wasm/nft_idb.rs @@ -27,7 +27,6 @@ impl DbInstance for NftCacheIDB { } } -#[allow(dead_code)] impl NftCacheIDB { pub(crate) fn get_inner(&self) -> &IndexedDb { &self.inner } } From 2ce92e43cac81848b6f43fa5709747c618b9b9d9 Mon Sep 17 00:00:00 2001 From: laruh Date: Sun, 25 Jun 2023 19:59:13 +0700 Subject: [PATCH 098/112] return bool in check_and_redact_metadata_field --- mm2src/coins/nft.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index ab73099687..faa745caa2 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -839,7 +839,7 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { if let Some(metadata_str) = &nft.metadata { if let Ok(mut metadata) = serde_json::from_str::>(metadata_str) { - if check_and_redact_metadata_field(&mut metadata, "name")? { + if check_and_redact_metadata_field(&mut metadata, "name") { nft.metadata = Some(serde_json::to_string(&metadata)?); return Ok(true); } @@ -848,18 +848,15 @@ fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult, - field: &str, -) -> MmResult { +fn check_and_redact_metadata_field(metadata: &mut serde_json::Map, field: &str) -> bool { match metadata.get(field).and_then(|v| v.as_str()) { Some(value) if value.contains("http://") || value.contains("https://") => { metadata.insert( field.to_string(), serde_json::Value::String("URL redacted for user protection".to_string()), ); - Ok(true) + true }, - _ => Ok(false), + _ => false, } } From 9b4b85e56dab5f906f403cf389828b8d171c3a6b Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 26 Jun 2023 17:20:11 +0700 Subject: [PATCH 099/112] if check in UriMeta::merge_from, line break after index_keys, get_last_block_from_table func --- mm2src/coins/nft/nft_structs.rs | 28 ++++-- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 98 +++++++++++-------- 2 files changed, 76 insertions(+), 50 deletions(-) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index bddbe6e855..081f436491 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -160,13 +160,27 @@ pub(crate) struct UriMeta { impl UriMeta { pub(crate) fn merge_from(&mut self, other: UriMetaFromStr) { - self.image = self.image.clone().or(other.image).or(other.image_url); - self.token_name = self.token_name.clone().or(other.token_name); - self.description = self.description.clone().or(other.description); - self.attributes = self.attributes.clone().or(other.attributes); - self.animation_url = self.animation_url.clone().or(other.animation_url); - self.external_url = self.external_url.clone().or(other.external_url); - self.image_details = self.image_details.clone().or(other.image_details); + if self.image.is_none() { + self.image = other.image.or(other.image_url); + } + if self.token_name.is_none() { + self.token_name = other.token_name; + } + if self.description.is_none() { + self.description = other.description; + } + if self.attributes.is_none() { + self.attributes = other.attributes; + } + if self.animation_url.is_none() { + self.animation_url = other.animation_url; + } + if self.external_url.is_none() { + self.external_url = other.external_url; + } + if self.image_details.is_none() { + self.image_details = other.image_details; + } } } diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 885e1c4279..297c98ed44 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -7,7 +7,7 @@ use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorag use crate::CoinsContext; use async_trait::async_trait; use mm2_core::mm_ctx::MmArc; -use mm2_db::indexed_db::{BeBigUint, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; +use mm2_db::indexed_db::{BeBigUint, DbTable, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_err_handle::prelude::MmResult; @@ -170,6 +170,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; + if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { Ok(Some(nft_details_from_item(item)?)) } else { @@ -193,6 +194,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; + let last_scanned_block = LastScannedBlockTable { chain: chain.to_string(), last_scanned_block: BeBigUint::from(scanned_block), @@ -222,6 +224,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; + if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { Ok(Some(nft_details_from_item(item)?.amount.to_string())) } else { @@ -233,11 +236,11 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&nft.token_address)? .with_value(nft.token_id.to_string())?; + let nft_item = NftListTable::from_nft(&nft)?; table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; Ok(()) @@ -247,27 +250,7 @@ impl NftListStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let maybe_item = table - .cursor_builder() - .only("chain", chain.to_string()) - .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .bound("block_number", BeBigUint::from(0u64), BeBigUint::from(u64::MAX)) - .reverse() - .open_cursor(NftListTable::CHAIN_BLOCK_NUMBER_INDEX) - .await - .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .next() - .await - .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; - let maybe_item = maybe_item - .map(|(_, item)| { - item.block_number - .to_u64() - .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string())) - }) - .transpose()?; - - Ok(maybe_item) + get_last_block_from_table(chain, table, NftListTable::CHAIN_BLOCK_NUMBER_INDEX).await } async fn get_last_scanned_block(&self, chain: &Chain) -> MmResult, Self::Error> { @@ -295,6 +278,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(chain.to_string())? .with_value(&nft.token_address)? .with_value(nft.token_id.to_string())?; + let nft_item = NftListTable::from_nft(&nft)?; nft_table .replace_item_by_unique_multi_index(index_keys, &nft_item) @@ -319,6 +303,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(chain.to_string())? .with_value(&nft.token_address)? .with_value(nft.token_id.to_string())?; + let nft_item = NftListTable::from_nft(&nft)?; nft_table .replace_item_by_unique_multi_index(index_keys, &nft_item) @@ -385,26 +370,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; - let maybe_item = table - .cursor_builder() - .only("chain", chain.to_string()) - .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .bound("block_number", BeBigUint::from(0u64), BeBigUint::from(u64::MAX)) - .reverse() - .open_cursor(NftTxHistoryTable::CHAIN_BLOCK_NUMBER_INDEX) - .await - .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? - .next() - .await - .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; - let maybe_item = maybe_item - .map(|(_, item)| { - item.block_number - .to_u64() - .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string())) - }) - .transpose()?; - Ok(maybe_item) + get_last_block_from_table(chain, table, NftTxHistoryTable::CHAIN_BLOCK_NUMBER_INDEX).await } async fn get_txs_from_block( @@ -446,10 +412,12 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? .with_value(&token_address)? .with_value(token_id.to_string())?; + table .get_items_by_multi_index(index_keys) .await? @@ -469,6 +437,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) .with_value(chain.to_string())? .with_value(&transaction_hash)?; + if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { Ok(Some(tx_details_from_item(item)?)) } else { @@ -480,9 +449,11 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let locked_db = self.lock_db().await?; let db_transaction = locked_db.get_inner().transaction().await?; let table = db_transaction.table::().await?; + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) .with_value(chain.to_string())? .with_value(&tx.transaction_hash)?; + let item = NftTxHistoryTable::from_tx_history(&tx)?; table.replace_item_by_unique_multi_index(index_keys, &item).await?; Ok(()) @@ -501,9 +472,11 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { tx.image = tx_meta.image.clone(); tx.token_name = tx_meta.token_name.clone(); drop_mutability!(tx); + let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) .with_value(chain.to_string())? .with_value(&tx.transaction_hash)?; + let item = NftTxHistoryTable::from_tx_history(&tx)?; table.replace_item_by_unique_multi_index(index_keys, &item).await?; } @@ -544,6 +517,45 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { } } +async fn get_last_block_from_table( + chain: &Chain, + table: DbTable<'_, impl TableSignature + BlockNumberTable>, + cursor: &str, +) -> MmResult, WasmNftCacheError> { + let maybe_item = table + .cursor_builder() + .only("chain", chain.to_string()) + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .bound("block_number", BeBigUint::from(0u64), BeBigUint::from(u64::MAX)) + .reverse() + .open_cursor(cursor) + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + .next() + .await + .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?; + let maybe_item = maybe_item + .map(|(_, item)| { + item.get_block_number() + .to_u64() + .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string())) + }) + .transpose()?; + Ok(maybe_item) +} + +trait BlockNumberTable { + fn get_block_number(&self) -> &BeBigUint; +} + +impl BlockNumberTable for NftListTable { + fn get_block_number(&self) -> &BeBigUint { &self.block_number } +} + +impl BlockNumberTable for NftTxHistoryTable { + fn get_block_number(&self) -> &BeBigUint { &self.block_number } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub(crate) struct NftListTable { token_address: String, From 0d8bebea8d3e374381e63de4452a794c19a73bc0 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 27 Jun 2023 22:05:50 +0700 Subject: [PATCH 100/112] fix review notes: doc comments added, UriMeta refactored, db test functions use block_on --- mm2src/coins/nft.rs | 89 +-- mm2src/coins/nft/nft_structs.rs | 53 +- mm2src/coins/nft/nft_tests.rs | 48 +- .../{for_db_tests.rs => db_test_helpers.rs} | 556 +++++++++--------- mm2src/coins/nft/storage/mod.rs | 3 +- mm2src/coins/nft/storage/sql_storage.rs | 14 +- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 27 +- 7 files changed, 417 insertions(+), 373 deletions(-) rename mm2src/coins/nft/storage/{for_db_tests.rs => db_test_helpers.rs} (50%) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index faa745caa2..5016fc786f 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -15,7 +15,7 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, N TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta, UriMetaFromStr}; +use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use http::header::ACCEPT; @@ -44,16 +44,16 @@ pub async fn get_nft_list(ctx: MmArc, req: NftListReq) -> MmResult MmResult MmResu token_id: nft_db.token_id, token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, + image_url: nft_db.uri_meta.image_url, token_name: nft_db.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(&nft_db.chain, tx_meta).await?; @@ -325,7 +326,7 @@ async fn get_moralis_nft_transfers( token_id: transfer_wrapper.token_id.0, token_uri: None, collection_name: None, - image: None, + image_url: None, token_name: None, from_address: transfer_wrapper.from_address, to_address: transfer_wrapper.to_address, @@ -476,27 +477,21 @@ fn check_moralis_ipfs_bafy(token_uri: Option<&str>) -> Option { }) } -async fn try_get_uri_meta(token_uri: Option<&str>) -> UriMeta { +async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>) -> UriMeta { let mut uri_meta = UriMeta::default(); if let Some(token_uri) = token_uri { if let Ok(response_meta) = send_request_to_uri(token_uri).await { - if let Ok(uri_meta_res) = serde_json::from_str::(&response_meta.to_string()) { - uri_meta.merge_from(uri_meta_res); + if let Ok(token_uri_meta) = serde_json::from_value(response_meta) { + uri_meta = token_uri_meta; } } } - drop_mutability!(uri_meta); - uri_meta -} - -async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>) -> UriMeta { - let mut uri_meta = try_get_uri_meta(token_uri).await; if let Some(metadata) = metadata { - if let Ok(meta_from_meta) = serde_json::from_str::(metadata) { - uri_meta.merge_from(meta_from_meta); + if let Ok(meta_from_meta) = serde_json::from_str::(metadata) { + uri_meta.merge_from(meta_from_meta) } } - uri_meta.image = check_moralis_ipfs_bafy(uri_meta.image.as_deref()); + uri_meta.image_url = check_moralis_ipfs_bafy(uri_meta.image_url.as_deref()); uri_meta.animation_url = check_moralis_ipfs_bafy(uri_meta.animation_url.as_deref()); drop_mutability!(uri_meta); uri_meta @@ -567,7 +562,7 @@ async fn handle_send_erc721( token_id: nft_db.token_id, token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, + image_url: nft_db.uri_meta.image_url, token_name: nft_db.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -598,7 +593,7 @@ async fn handle_receive_erc721( token_id: nft.token_id, token_uri: nft.token_uri, collection_name: nft.collection_name, - image: nft.uri_meta.image, + image_url: nft.uri_meta.image_url, token_name: nft.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -624,7 +619,7 @@ async fn handle_send_erc1155( token_id: nft_db.token_id, token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, + image_url: nft_db.uri_meta.image_url, token_name: nft_db.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -643,7 +638,7 @@ async fn handle_send_erc1155( token_id: nft_db.token_id, token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, + image_url: nft_db.uri_meta.image_url, token_name: nft_db.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -685,7 +680,7 @@ async fn handle_receive_erc1155( token_id: nft_db.token_id, token_uri: nft_db.token_uri, collection_name: nft_db.collection_name, - image: nft_db.uri_meta.image, + image_url: nft_db.uri_meta.image_url, token_name: nft_db.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -719,7 +714,7 @@ async fn handle_receive_erc1155( token_id: nft.token_id, token_uri: nft.token_uri, collection_name: nft.collection_name, - image: nft.uri_meta.image, + image_url: nft.uri_meta.image_url, token_name: nft.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -776,7 +771,7 @@ where token_id: nft.token_id, token_uri: nft.token_uri, collection_name: nft.collection_name, - image: nft.uri_meta.image, + image_url: nft.uri_meta.image_url, token_name: nft.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -797,7 +792,7 @@ where token_id: nft_meta.token_id, token_uri: nft_meta.token_uri, collection_name: nft_meta.collection_name, - image: nft_meta.uri_meta.image, + image_url: nft_meta.uri_meta.image_url, token_name: nft_meta.uri_meta.token_name, }; storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; @@ -805,25 +800,35 @@ where Ok(()) } -fn check_and_redact_if_spam(value: &mut Option) -> bool { - match value { +/// `check_and_redact_if_spam` checks if the text contains any links./// +/// It doesn't matter if the link is valid or not, as this is a spam check. +/// If text contains some link, then it is a spam. +fn check_and_redact_if_spam(text: &mut Option) -> bool { + match text { Some(s) if s.contains("http://") || s.contains("https://") => { - *value = Some("URL redacted for user protection".to_string()); + *text = Some("URL redacted for user protection".to_string()); true }, _ => false, } } - +/// `protect_from_history_spam` function checks and redact spam in `NftTransferHistory`. +/// +/// `collection_name` and `token_name` in `NftTransferHistory` shouldn't contain any links, +/// they must be just an arbitrary text, which represents NFT names. fn protect_from_history_spam(tx: &mut NftTransferHistory) { let collection_name_spam = check_and_redact_if_spam(&mut tx.collection_name); let token_name_spam = check_and_redact_if_spam(&mut tx.token_name); if collection_name_spam || token_name_spam { - tx.possible_spam = Some(true); + tx.possible_spam = true; } } - +/// `protect_from_nft_spam` function checks and redact spam in `Nft`. +/// +/// `collection_name` and `token_name` in `Nft` shouldn't contain any links, +/// they must be just an arbitrary text, which represents NFT names. +/// `symbol` also must be a text or sign that represents a symbol. fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { let collection_name_spam = check_and_redact_if_spam(&mut nft.collection_name); let symbol_spam = check_and_redact_if_spam(&mut nft.symbol); @@ -831,15 +836,17 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { let meta_spam = check_nft_metadata_for_spam(nft)?; if collection_name_spam || symbol_spam || token_name_spam || meta_spam { - nft.possible_spam = Some(true); + nft.possible_spam = true; } Ok(()) } - +/// `check_nft_metadata_for_spam` function checks and redact spam in `metadata` field from `Nft`. +/// +/// **note:** `token_name` is usually called `name` in `metadata`. fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { if let Some(metadata_str) = &nft.metadata { if let Ok(mut metadata) = serde_json::from_str::>(metadata_str) { - if check_and_redact_metadata_field(&mut metadata, "name") { + if check_spam_and_redact_metadata_field(&mut metadata, "name") { nft.metadata = Some(serde_json::to_string(&metadata)?); return Ok(true); } @@ -848,9 +855,15 @@ fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult, field: &str) -> bool { +/// The `check_spam_and_redact_metadata_field` function scans a specified field in a JSON metadata object for potential spam. +/// +/// This function checks the provided `metadata` map for a field matching the `field` parameter. +/// If this field is found and its value contains some link, it's considered to contain spam. +/// To protect users, function redacts field containing spam link. +/// The function returns `true` if it detected spam link, or `false` otherwise. +fn check_spam_and_redact_metadata_field(metadata: &mut serde_json::Map, field: &str) -> bool { match metadata.get(field).and_then(|v| v.as_str()) { - Some(value) if value.contains("http://") || value.contains("https://") => { + Some(text) if text.contains("http://") || text.contains("https://") => { metadata.insert( field.to_string(), serde_json::Value::String("URL redacted for user protection".to_string()), diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 081f436491..4436316451 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -132,24 +132,21 @@ impl fmt::Display for ContractType { } } -#[derive(Clone, Debug, Default, Deserialize, Serialize)] -pub(crate) struct UriMetaFromStr { - pub(crate) image: Option, - /// the same as image, some tokens use **image_url** name for **image** field - /// but sometimes token_uri contains both`image_url` and `image` - pub(crate) image_url: Option, - #[serde(rename(deserialize = "name"))] - pub(crate) token_name: Option, - pub(crate) description: Option, - pub(crate) attributes: Option, - pub(crate) animation_url: Option, - pub(crate) external_url: Option, - pub(crate) image_details: Option, -} - +/// `UriMeta` structure is the object which we create from `token_uri` and `metadata`. +/// +/// `token_uri` and `metadata` usually contain either `image` or `image_url` with image url. +/// But most often nft creators use only `image` name for this value (from my observation), +/// less often they use both parameters with the same url. +/// +/// I suspect this is because some APIs only look for one of these image url names, so nft creators try to satisfy all sides. +/// In any case, since there is no clear standard, we have to look for both options, +/// when we build `UriMeta` from `token_uri` or `metadata`. #[derive(Clone, Debug, Default, Deserialize, Serialize)] pub(crate) struct UriMeta { - pub(crate) image: Option, + #[serde(rename = "image")] + pub(crate) raw_image_url: Option, + pub(crate) image_url: Option, + #[serde(rename = "name")] pub(crate) token_name: Option, pub(crate) description: Option, pub(crate) attributes: Option, @@ -159,9 +156,13 @@ pub(crate) struct UriMeta { } impl UriMeta { - pub(crate) fn merge_from(&mut self, other: UriMetaFromStr) { - if self.image.is_none() { - self.image = other.image.or(other.image_url); + /// `merge_from` function doesnt change `raw_image_url` field. + /// It tries to update `image_url` field instead, if it is None. + /// As `image` is the original name of `raw_image_url` field in data from `token_uri` or `metadata`, + /// try to find **Some()** in this field first. + pub(crate) fn merge_from(&mut self, other: UriMeta) { + if self.image_url.is_none() { + self.image_url = other.raw_image_url.or(other.image_url); } if self.token_name.is_none() { self.token_name = other.token_name; @@ -202,7 +203,7 @@ pub struct Nft { pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, - pub(crate) possible_spam: Option, + pub(crate) possible_spam: bool, pub(crate) uri_meta: UriMeta, } @@ -225,7 +226,8 @@ pub(crate) struct NftWrapper { pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, - pub(crate) possible_spam: Option, + #[serde(default)] + pub(crate) possible_spam: bool, } #[derive(Debug)] @@ -377,7 +379,7 @@ pub struct NftTransferHistory { pub(crate) token_id: BigDecimal, pub(crate) token_uri: Option, pub(crate) collection_name: Option, - pub(crate) image: Option, + pub(crate) image_url: Option, pub(crate) token_name: Option, pub(crate) from_address: String, pub(crate) to_address: String, @@ -385,7 +387,7 @@ pub struct NftTransferHistory { pub(crate) amount: BigDecimal, pub(crate) verified: u64, pub(crate) operator: Option, - pub(crate) possible_spam: Option, + pub(crate) possible_spam: bool, } #[derive(Debug, Deserialize)] @@ -407,7 +409,8 @@ pub(crate) struct NftTransferHistoryWrapper { pub(crate) amount: SerdeStringWrap, pub(crate) verified: u64, pub(crate) operator: Option, - pub(crate) possible_spam: Option, + #[serde(default)] + pub(crate) possible_spam: bool, } #[derive(Debug, Serialize)] @@ -445,6 +448,6 @@ pub struct TxMeta { pub(crate) token_id: BigDecimal, pub(crate) token_uri: Option, pub(crate) collection_name: Option, - pub(crate) image: Option, + pub(crate) image_url: Option, pub(crate) token_name: Option, } diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 7ac4f66394..785fcce48e 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -7,7 +7,7 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; mod native_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; - use crate::nft::storage::for_db_tests::*; + use crate::nft::storage::db_test_helpers::*; use crate::nft::{check_moralis_ipfs_bafy, send_request_to_uri}; use common::block_on; @@ -51,37 +51,37 @@ mod native_tests { } #[test] - fn test_add_get_nfts() { block_on(test_add_get_nfts_impl()) } + fn test_add_get_nfts() { test_add_get_nfts_impl() } #[test] - fn test_last_nft_blocks() { block_on(test_last_nft_blocks_impl()) } + fn test_last_nft_blocks() { test_last_nft_blocks_impl() } #[test] - fn test_nft_list() { block_on(test_nft_list_impl()) } + fn test_nft_list() { test_nft_list_impl() } #[test] - fn test_remove_nft() { block_on(test_remove_nft_impl()) } + fn test_remove_nft() { test_remove_nft_impl() } #[test] - fn test_refresh_metadata() { block_on(test_refresh_metadata_impl()) } + fn test_refresh_metadata() { test_refresh_metadata_impl() } #[test] - fn test_nft_amount() { block_on(test_nft_amount_impl()) } + fn test_nft_amount() { test_nft_amount_impl() } #[test] - fn test_add_get_txs() { block_on(test_add_get_txs_impl()) } + fn test_add_get_txs() { test_add_get_txs_impl() } #[test] - fn test_last_tx_block() { block_on(test_last_tx_block_impl()) } + fn test_last_tx_block() { test_last_tx_block_impl() } #[test] - fn test_tx_history() { block_on(test_tx_history_impl()) } + fn test_tx_history() { test_tx_history_impl() } #[test] - fn test_tx_history_filters() { block_on(test_tx_history_filters_impl()) } + fn test_tx_history_filters() { test_tx_history_filters_impl() } #[test] - fn test_get_update_tx_meta() { block_on(test_get_update_tx_meta_impl()) } + fn test_get_update_tx_meta() { test_get_update_tx_meta_impl() } } #[cfg(target_arch = "wasm32")] @@ -89,7 +89,7 @@ mod wasm_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::send_request_to_uri; - use crate::nft::storage::for_db_tests::*; + use crate::nft::storage::db_test_helpers::*; use wasm_bindgen_test::*; wasm_bindgen_test_configure!(run_in_browser); @@ -122,35 +122,35 @@ mod wasm_tests { } #[wasm_bindgen_test] - async fn test_add_get_nfts() { test_add_get_nfts_impl().await } + fn test_add_get_nfts() { test_add_get_nfts_impl() } #[wasm_bindgen_test] - async fn test_last_nft_blocks() { test_last_nft_blocks_impl().await } + fn test_last_nft_blocks() { test_last_nft_blocks_impl() } #[wasm_bindgen_test] - async fn test_nft_list() { test_nft_list_impl().await } + fn test_nft_list() { test_nft_list_impl() } #[wasm_bindgen_test] - async fn test_remove_nft() { test_remove_nft_impl().await } + fn test_remove_nft() { test_remove_nft_impl() } #[wasm_bindgen_test] - async fn test_nft_amount() { test_nft_amount_impl().await } + fn test_nft_amount() { test_nft_amount_impl() } #[wasm_bindgen_test] - async fn test_refresh_metadata() { test_refresh_metadata_impl().await } + fn test_refresh_metadata() { test_refresh_metadata_impl() } #[wasm_bindgen_test] - async fn test_add_get_txs() { test_add_get_txs_impl().await } + fn test_add_get_txs() { test_add_get_txs_impl() } #[wasm_bindgen_test] - async fn test_last_tx_block() { test_last_tx_block_impl().await } + fn test_last_tx_block() { test_last_tx_block_impl() } #[wasm_bindgen_test] - async fn test_tx_history() { test_tx_history_impl().await } + fn test_tx_history() { test_tx_history_impl() } #[wasm_bindgen_test] - async fn test_tx_history_filters() { test_tx_history_filters_impl().await } + fn test_tx_history_filters() { test_tx_history_filters_impl() } #[wasm_bindgen_test] - async fn test_get_update_tx_meta() { test_get_update_tx_meta_impl().await } + fn test_get_update_tx_meta() { test_get_update_tx_meta_impl() } } diff --git a/mm2src/coins/nft/storage/for_db_tests.rs b/mm2src/coins/nft/storage/db_test_helpers.rs similarity index 50% rename from mm2src/coins/nft/storage/for_db_tests.rs rename to mm2src/coins/nft/storage/db_test_helpers.rs index 0b036429cb..80e88e1f43 100644 --- a/mm2src/coins/nft/storage/for_db_tests.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -1,6 +1,7 @@ use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; +use common::block_on; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; use std::num::NonZeroUsize; @@ -34,9 +35,10 @@ fn nft() -> Nft { last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: Some(false), + possible_spam: false, uri_meta: UriMeta { - image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + raw_image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), token_name: None, description: Some("Born to usher in Bull markets.".to_string()), attributes: None, @@ -63,7 +65,7 @@ fn tx() -> NftTransferHistory { token_id: BigDecimal::from_str("214300047252").unwrap(), token_uri: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), token_name: Some("Nebula Nodes".to_string()), from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), @@ -71,7 +73,7 @@ fn tx() -> NftTransferHistory { amount: BigDecimal::from_str("1").unwrap(), verified: 1, operator: None, - possible_spam: Some(false), + possible_spam: false, } } @@ -93,9 +95,10 @@ fn nft_list() -> Vec { last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: Some(false), + possible_spam: false, uri_meta: UriMeta { - image: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), + raw_image_url: None, token_name: None, description: Some("Born to usher in Bull markets.".to_string()), attributes: None, @@ -125,9 +128,12 @@ fn nft_list() -> Vec { last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: Some(false), + possible_spam: false, uri_meta: UriMeta { - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + image_url: Some( + "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), + ), + raw_image_url: None, token_name: Some("Nebula Nodes".to_string()), description: Some("Interchain nodes".to_string()), attributes: None, @@ -157,9 +163,12 @@ fn nft_list() -> Vec { last_token_uri_sync: Some("2023-02-19T19:12:09.471Z".to_string()), last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: Some(false), + possible_spam: false, uri_meta: UriMeta { - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + image_url: Some( + "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), + ), + raw_image_url: None, token_name: Some("Nebula Nodes".to_string()), description: Some("Interchain nodes".to_string()), attributes: None, @@ -187,7 +196,7 @@ fn nft_tx_historty() -> Vec { token_id: Default::default(), token_uri: None, collection_name: None, - image: None, + image_url: None, token_name: None, from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), @@ -195,7 +204,7 @@ fn nft_tx_historty() -> Vec { amount: BigDecimal::from_str("1").unwrap(), verified: 1, operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), - possible_spam: Some(false), + possible_spam: false, }; let tx1 = NftTransferHistory { @@ -213,7 +222,7 @@ fn nft_tx_historty() -> Vec { token_id: BigDecimal::from_str("214300047252").unwrap(), token_uri: None, collection_name: None, - image: None, + image_url: None, token_name: None, from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), @@ -221,7 +230,7 @@ fn nft_tx_historty() -> Vec { amount: BigDecimal::from_str("1").unwrap(), verified: 1, operator: None, - possible_spam: Some(false), + possible_spam: false, }; let tx2 = NftTransferHistory { @@ -239,7 +248,7 @@ fn nft_tx_historty() -> Vec { token_id: BigDecimal::from_str("214300044414").unwrap(), token_uri: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - image: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), + image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), token_name: Some("Nebula Nodes".to_string()), from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), @@ -247,7 +256,7 @@ fn nft_tx_historty() -> Vec { amount: BigDecimal::from_str("1").unwrap(), verified: 1, operator: None, - possible_spam: Some(false), + possible_spam: false, }; vec![tx, tx1, tx2] } @@ -270,270 +279,293 @@ async fn init_nft_history_storage(chain: &Chain) -> impl NftListStorageOps + Nft storage } -pub(crate) async fn test_add_get_nfts_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(nft.block_number, 28056721); +pub(crate) fn test_add_get_nfts_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); + }); } -pub(crate) async fn test_last_nft_blocks_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - let last_block = NftListStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_scanned_block, last_block); +pub(crate) fn test_last_nft_blocks_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); + }); } -pub(crate) async fn test_nft_list_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let nft_list = storage - .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 1); - let nft = nft_list.nfts.get(0).unwrap(); - assert_eq!(nft.block_number, 28056721); - assert_eq!(nft_list.skipped, 1); - assert_eq!(nft_list.total, 3); +pub(crate) fn test_nft_list_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let nft_list = storage + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 1); + let nft = nft_list.nfts.get(0).unwrap(); + assert_eq!(nft.block_number, 28056721); + assert_eq!(nft_list.skipped, 1); + assert_eq!(nft_list.total, 3); + }); } -pub(crate) async fn test_remove_nft_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let remove_rslt = storage - .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) - .await - .unwrap(); - assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); - let list_len = storage - .get_nft_list(vec![chain], true, 1, None) - .await - .unwrap() - .nfts - .len(); - assert_eq!(list_len, 2); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 28056800); +pub(crate) fn test_remove_nft_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let remove_rslt = storage + .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) + .await + .unwrap(); + assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); + let list_len = storage + .get_nft_list(vec![chain], true, 1, None) + .await + .unwrap() + .nfts + .len(); + assert_eq!(list_len, 2); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 28056800); + }); } -pub(crate) async fn test_nft_amount_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let mut nft = nft(); - storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - - nft.amount -= BigDecimal::from(1); - storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); - let amount = storage - .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "1"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919800); - - nft.amount += BigDecimal::from(1); - nft.block_number = 25919900; - storage - .update_nft_amount_and_block_number(&chain, nft.clone()) - .await - .unwrap(); - let amount = storage - .get_nft_amount(&chain, nft.token_address, nft.token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "2"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919900); +pub(crate) fn test_nft_amount_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + + nft.amount -= BigDecimal::from(1); + storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "1"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919800); + + nft.amount += BigDecimal::from(1); + nft.block_number = 25919900; + storage + .update_nft_amount_and_block_number(&chain, nft.clone()) + .await + .unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address, nft.token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "2"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919900); + }); } -pub(crate) async fn test_refresh_metadata_impl() { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let new_symbol = "NEW_SYMBOL"; - let mut nft = nft(); - storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - nft.symbol = Some(new_symbol.to_string()); - drop_mutability!(nft); - let token_add = nft.token_address.clone(); - let token_id = nft.token_id.clone(); - storage.refresh_nft_metadata(&chain, nft).await.unwrap(); - let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); - assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); +pub(crate) fn test_refresh_metadata_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let new_symbol = "NEW_SYMBOL"; + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + nft.symbol = Some(new_symbol.to_string()); + drop_mutability!(nft); + let token_add = nft.token_address.clone(); + let token_id = nft.token_id.clone(); + storage.refresh_nft_metadata(&chain, nft).await.unwrap(); + let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); + }); } -pub(crate) async fn test_add_get_txs_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let tx1 = storage - .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .get(0) - .unwrap() - .clone(); - assert_eq!(tx1.block_number, 28056721); - let tx2 = storage - .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(tx2.block_number, 28056726); - let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); - assert_eq!(tx_from.len(), 2); +pub(crate) fn test_add_get_txs_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let tx1 = storage + .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .get(0) + .unwrap() + .clone(); + assert_eq!(tx1.block_number, 28056721); + let tx2 = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(tx2.block_number, 28056726); + let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); + assert_eq!(tx_from.len(), 2); + }); } -pub(crate) async fn test_last_tx_block_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); +pub(crate) fn test_last_tx_block_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); + }); } -pub(crate) async fn test_tx_history_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let tx_history = storage - .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) - .await - .unwrap(); - assert_eq!(tx_history.transfer_history.len(), 1); - let tx = tx_history.transfer_history.get(0).unwrap(); - assert_eq!(tx.block_number, 28056721); - assert_eq!(tx_history.skipped, 1); - assert_eq!(tx_history.total, 3); +pub(crate) fn test_tx_history_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let tx_history = storage + .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 1); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056721); + assert_eq!(tx_history.skipped, 1); + assert_eq!(tx_history.total, 3); + }); } -pub(crate) async fn test_tx_history_filters_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let filters = NftTxHistoryFilters { - receive: true, - send: false, - from_date: None, - to_date: None, - }; - - let filters1 = NftTxHistoryFilters { - receive: false, - send: false, - from_date: None, - to_date: Some(1677166110), - }; - - let filters2 = NftTxHistoryFilters { - receive: false, - send: false, - from_date: Some(1677166110), - to_date: Some(1683627417), - }; - - let tx_history = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(tx_history.transfer_history.len(), 3); - let tx = tx_history.transfer_history.get(0).unwrap(); - assert_eq!(tx.block_number, 28056726); - - let tx_history1 = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters1)) - .await - .unwrap(); - assert_eq!(tx_history1.transfer_history.len(), 1); - let tx1 = tx_history1.transfer_history.get(0).unwrap(); - assert_eq!(tx1.block_number, 25919780); - - let tx_history2 = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters2)) - .await - .unwrap(); - assert_eq!(tx_history2.transfer_history.len(), 2); - let tx_0 = tx_history2.transfer_history.get(0).unwrap(); - assert_eq!(tx_0.block_number, 28056721); - let tx_1 = tx_history2.transfer_history.get(1).unwrap(); - assert_eq!(tx_1.block_number, 25919780); +pub(crate) fn test_tx_history_filters_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let filters = NftTxHistoryFilters { + receive: true, + send: false, + from_date: None, + to_date: None, + }; + + let filters1 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: None, + to_date: Some(1677166110), + }; + + let filters2 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: Some(1677166110), + to_date: Some(1683627417), + }; + + let tx_history = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 3); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056726); + + let tx_history1 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap(); + assert_eq!(tx_history1.transfer_history.len(), 1); + let tx1 = tx_history1.transfer_history.get(0).unwrap(); + assert_eq!(tx1.block_number, 25919780); + + let tx_history2 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters2)) + .await + .unwrap(); + assert_eq!(tx_history2.transfer_history.len(), 2); + let tx_0 = tx_history2.transfer_history.get(0).unwrap(); + assert_eq!(tx_0.block_number, 28056721); + let tx_1 = tx_history2.transfer_history.get(1).unwrap(); + assert_eq!(tx_1.block_number, 25919780); + }); } -pub(crate) async fn test_get_update_tx_meta_impl() { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); - assert_eq!(vec_token_add_id.len(), 2); - - let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); - let tx_meta = TxMeta { - token_address: token_add.clone(), - token_id: Default::default(), - token_uri: None, - collection_name: None, - image: None, - token_name: Some("Tiki box".to_string()), - }; - storage.update_txs_meta_by_token_addr_id(&chain, tx_meta).await.unwrap(); - let tx_upd = storage - .get_txs_by_token_addr_id(&chain, token_add, Default::default()) - .await - .unwrap(); - let tx_upd = tx_upd.get(0).unwrap(); - assert_eq!(tx_upd.token_name, Some("Tiki box".to_string())); - - let tx_meta = tx(); - storage.update_tx_meta_by_hash(&chain, tx_meta).await.unwrap(); - let tx_by_hash = storage - .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(tx_by_hash.token_name, Some("Nebula Nodes".to_string())) +pub(crate) fn test_get_update_tx_meta_impl() { + block_on(async { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); + assert_eq!(vec_token_add_id.len(), 2); + + let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); + let tx_meta = TxMeta { + token_address: token_add.clone(), + token_id: Default::default(), + token_uri: None, + collection_name: None, + image_url: None, + token_name: Some("Tiki box".to_string()), + }; + storage.update_txs_meta_by_token_addr_id(&chain, tx_meta).await.unwrap(); + let tx_upd = storage + .get_txs_by_token_addr_id(&chain, token_add, Default::default()) + .await + .unwrap(); + let tx_upd = tx_upd.get(0).unwrap(); + assert_eq!(tx_upd.token_name, Some("Tiki box".to_string())); + + let tx_meta = tx(); + storage.update_tx_meta_by_hash(&chain, tx_meta).await.unwrap(); + let tx_by_hash = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(tx_by_hash.token_name, Some("Nebula Nodes".to_string())) + }); } diff --git a/mm2src/coins/nft/storage/mod.rs b/mm2src/coins/nft/storage/mod.rs index dca889189d..c48c798875 100644 --- a/mm2src/coins/nft/storage/mod.rs +++ b/mm2src/coins/nft/storage/mod.rs @@ -11,7 +11,7 @@ use serde::{Deserialize, Serialize}; use std::num::NonZeroUsize; #[cfg(any(test, target_arch = "wasm32"))] -pub(crate) mod for_db_tests; +pub(crate) mod db_test_helpers; #[cfg(not(target_arch = "wasm32"))] pub(crate) mod sql_storage; #[cfg(target_arch = "wasm32")] pub(crate) mod wasm; @@ -173,6 +173,7 @@ impl<'a> NftStorageBuilder<'a> { } } +/// `get_offset_limit` function calculates offset and limit for final result if we use pagination. fn get_offset_limit(max: bool, limit: usize, page_number: Option, total_count: usize) -> (usize, usize) { if max { return (0, total_count); diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index b7b0faadae..6705a2ab8f 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -60,7 +60,7 @@ fn create_tx_history_table_sql(chain: &Chain) -> MmResult { amount VARCHAR(256) NOT NULL, token_uri TEXT, collection_name TEXT, - image TEXT, + image_url TEXT, token_name TEXT, details_json TEXT );", @@ -237,7 +237,7 @@ fn insert_tx_in_history_sql(chain: &Chain) -> MmResult { let sql = format!( "INSERT INTO {} ( transaction_hash, chain, block_number, block_timestamp, contract_type, - token_address, token_id, status, amount, collection_name, image, token_name, details_json + token_address, token_id, status, amount, collection_name, image_url, token_name, details_json ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13 );", @@ -275,7 +275,7 @@ fn update_meta_by_tx_hash_sql(chain: &Chain) -> MmResult { validate_table_name(&table_name)?; let sql = format!( - "UPDATE {} SET token_uri = ?1, collection_name = ?2, image = ?3, token_name = ?4, details_json = ?5 WHERE transaction_hash = ?6;", + "UPDATE {} SET token_uri = ?1, collection_name = ?2, image_url = ?3, token_name = ?4, details_json = ?5 WHERE transaction_hash = ?6;", table_name ); Ok(sql) @@ -412,7 +412,7 @@ fn get_txs_with_empty_meta_builder<'a>(conn: &'a Connection, chain: &'a Chain) - .field("token_id") .and_where_is_null("token_uri") .and_where_is_null("collection_name") - .and_where_is_null("image") + .and_where_is_null("image_url") .and_where_is_null("token_name"); drop_mutability!(sql_builder); Ok(sql_builder) @@ -765,7 +765,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { Some(tx.status.to_string()), Some(tx.amount.to_string()), tx.collection_name, - tx.image, + tx.image_url, tx.token_name, Some(tx_json), ]; @@ -843,7 +843,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { let params = [ tx.token_uri, tx.collection_name, - tx.image, + tx.image_url, tx.token_name, Some(tx_json), Some(tx.transaction_hash), @@ -867,7 +867,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { for mut tx in txs.into_iter() { tx.token_uri = tx_meta.token_uri.clone(); tx.collection_name = tx_meta.collection_name.clone(); - tx.image = tx_meta.image.clone(); + tx.image_url = tx_meta.image_url.clone(); tx.token_name = tx_meta.token_name.clone(); drop_mutability!(tx); selfi.update_tx_meta_by_hash(chain, tx).await?; diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 297c98ed44..37035879b9 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -85,18 +85,15 @@ impl IndexedDbNftStorage { filtered_txs.push(tx); } } - drop_mutability!(filtered_txs); Ok(filtered_txs) } } impl NftTxHistoryFilters { fn is_status_match(&self, tx: &NftTransferHistory) -> bool { - if !self.receive && !self.send { - true - } else { - (self.receive && tx.status == TransferStatus::Receive) || (self.send && tx.status == TransferStatus::Send) - } + (!self.receive && !self.send) + || (self.receive && tx.status == TransferStatus::Receive) + || (self.send && tx.status == TransferStatus::Send) } fn is_date_match(&self, tx: &NftTransferHistory) -> bool { @@ -174,7 +171,7 @@ impl NftListStorageOps for IndexedDbNftStorage { if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { Ok(Some(nft_details_from_item(item)?)) } else { - return Ok(None); + Ok(None) } } @@ -228,7 +225,7 @@ impl NftListStorageOps for IndexedDbNftStorage { if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { Ok(Some(nft_details_from_item(item)?.amount.to_string())) } else { - return Ok(None); + Ok(None) } } @@ -264,7 +261,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .ok_or_else(|| WasmNftCacheError::GetLastNftBlockError("height is too large".to_string()))?; Ok(Some(last_scanned_block)) } else { - return Ok(None); + Ok(None) } } @@ -399,7 +396,6 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let tx = tx_details_from_item(item)?; res.push(tx); } - drop_mutability!(res); Ok(res) } @@ -441,7 +437,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { Ok(Some(tx_details_from_item(item)?)) } else { - return Ok(None); + Ok(None) } } @@ -469,7 +465,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { for mut tx in txs { tx.token_uri = tx_meta.token_uri.clone(); tx.collection_name = tx_meta.collection_name.clone(); - tx.image = tx_meta.image.clone(); + tx.image_url = tx_meta.image_url.clone(); tx.token_name = tx_meta.token_name.clone(); drop_mutability!(tx); @@ -503,7 +499,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { { if item.token_uri.is_none() && item.collection_name.is_none() - && item.image.is_none() + && item.image_url.is_none() && item.token_name.is_none() { res.insert(NftTokenAddrId { @@ -512,7 +508,6 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { }); } } - drop_mutability!(res); Ok(res.into_iter().collect()) } } @@ -618,7 +613,7 @@ pub(crate) struct NftTxHistoryTable { amount: String, token_uri: Option, collection_name: Option, - image: Option, + image_url: Option, token_name: Option, details_json: Json, } @@ -644,7 +639,7 @@ impl NftTxHistoryTable { amount: tx.amount.to_string(), token_uri: tx.token_uri.clone(), collection_name: tx.collection_name.clone(), - image: tx.image.clone(), + image_url: tx.image_url.clone(), token_name: tx.token_name.clone(), details_json, }) From e869b0c4f40eb85c0ec143488b6128834162654f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 27 Jun 2023 22:43:07 +0700 Subject: [PATCH 101/112] change block_on --- mm2src/coins/nft/storage/db_test_helpers.rs | 23 ++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 80e88e1f43..aceccaf9cc 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -1,7 +1,6 @@ use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; -use common::block_on; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; use std::num::NonZeroUsize; @@ -280,7 +279,7 @@ async fn init_nft_history_storage(chain: &Chain) -> impl NftListStorageOps + Nft } pub(crate) fn test_add_get_nfts_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); @@ -297,7 +296,7 @@ pub(crate) fn test_add_get_nfts_impl() { } pub(crate) fn test_last_nft_blocks_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); @@ -314,7 +313,7 @@ pub(crate) fn test_last_nft_blocks_impl() { } pub(crate) fn test_nft_list_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); @@ -333,7 +332,7 @@ pub(crate) fn test_nft_list_impl() { } pub(crate) fn test_remove_nft_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let nft_list = nft_list(); @@ -358,7 +357,7 @@ pub(crate) fn test_remove_nft_impl() { } pub(crate) fn test_nft_amount_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let mut nft = nft(); @@ -396,7 +395,7 @@ pub(crate) fn test_nft_amount_impl() { } pub(crate) fn test_refresh_metadata_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_list_storage(&chain).await; let new_symbol = "NEW_SYMBOL"; @@ -416,7 +415,7 @@ pub(crate) fn test_refresh_metadata_impl() { } pub(crate) fn test_add_get_txs_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); @@ -443,7 +442,7 @@ pub(crate) fn test_add_get_txs_impl() { } pub(crate) fn test_last_tx_block_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); @@ -458,7 +457,7 @@ pub(crate) fn test_last_tx_block_impl() { } pub(crate) fn test_tx_history_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); @@ -477,7 +476,7 @@ pub(crate) fn test_tx_history_impl() { } pub(crate) fn test_tx_history_filters_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); @@ -533,7 +532,7 @@ pub(crate) fn test_tx_history_filters_impl() { } pub(crate) fn test_get_update_tx_meta_impl() { - block_on(async { + futures::executor::block_on(async { let chain = Chain::Bsc; let storage = init_nft_history_storage(&chain).await; let txs = nft_tx_historty(); From e8e13de08cfd6973899de2ad7b783e270837e61c Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 27 Jun 2023 23:19:17 +0700 Subject: [PATCH 102/112] use common::block_on for native tests --- mm2src/coins/nft/nft_tests.rs | 44 +- mm2src/coins/nft/storage/db_test_helpers.rs | 516 ++++++++++---------- 2 files changed, 269 insertions(+), 291 deletions(-) diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 785fcce48e..12da8d3c9c 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -51,37 +51,37 @@ mod native_tests { } #[test] - fn test_add_get_nfts() { test_add_get_nfts_impl() } + fn test_add_get_nfts() { block_on(test_add_get_nfts_impl()) } #[test] - fn test_last_nft_blocks() { test_last_nft_blocks_impl() } + fn test_last_nft_blocks() { block_on(test_last_nft_blocks_impl()) } #[test] - fn test_nft_list() { test_nft_list_impl() } + fn test_nft_list() { block_on(test_nft_list_impl()) } #[test] - fn test_remove_nft() { test_remove_nft_impl() } + fn test_remove_nft() { block_on(test_remove_nft_impl()) } #[test] - fn test_refresh_metadata() { test_refresh_metadata_impl() } + fn test_refresh_metadata() { block_on(test_refresh_metadata_impl()) } #[test] - fn test_nft_amount() { test_nft_amount_impl() } + fn test_nft_amount() { block_on(test_nft_amount_impl()) } #[test] - fn test_add_get_txs() { test_add_get_txs_impl() } + fn test_add_get_txs() { block_on(test_add_get_txs_impl()) } #[test] - fn test_last_tx_block() { test_last_tx_block_impl() } + fn test_last_tx_block() { block_on(test_last_tx_block_impl()) } #[test] - fn test_tx_history() { test_tx_history_impl() } + fn test_tx_history() { block_on(test_tx_history_impl()) } #[test] - fn test_tx_history_filters() { test_tx_history_filters_impl() } + fn test_tx_history_filters() { block_on(test_tx_history_filters_impl()) } #[test] - fn test_get_update_tx_meta() { test_get_update_tx_meta_impl() } + fn test_get_update_tx_meta() { block_on(test_get_update_tx_meta_impl()) } } #[cfg(target_arch = "wasm32")] @@ -122,35 +122,35 @@ mod wasm_tests { } #[wasm_bindgen_test] - fn test_add_get_nfts() { test_add_get_nfts_impl() } + async fn test_add_get_nfts() { test_add_get_nfts_impl().await } #[wasm_bindgen_test] - fn test_last_nft_blocks() { test_last_nft_blocks_impl() } + async fn test_last_nft_blocks() { test_last_nft_blocks_impl().await } #[wasm_bindgen_test] - fn test_nft_list() { test_nft_list_impl() } + async fn test_nft_list() { test_nft_list_impl().await } #[wasm_bindgen_test] - fn test_remove_nft() { test_remove_nft_impl() } + async fn test_remove_nft() { test_remove_nft_impl().await } #[wasm_bindgen_test] - fn test_nft_amount() { test_nft_amount_impl() } + async fn test_nft_amount() { test_nft_amount_impl().await } #[wasm_bindgen_test] - fn test_refresh_metadata() { test_refresh_metadata_impl() } + async fn test_refresh_metadata() { test_refresh_metadata_impl().await } #[wasm_bindgen_test] - fn test_add_get_txs() { test_add_get_txs_impl() } + async fn test_add_get_txs() { test_add_get_txs_impl().await } #[wasm_bindgen_test] - fn test_last_tx_block() { test_last_tx_block_impl() } + async fn test_last_tx_block() { test_last_tx_block_impl().await } #[wasm_bindgen_test] - fn test_tx_history() { test_tx_history_impl() } + async fn test_tx_history() { test_tx_history_impl().await } #[wasm_bindgen_test] - fn test_tx_history_filters() { test_tx_history_filters_impl() } + async fn test_tx_history_filters() { test_tx_history_filters_impl().await } #[wasm_bindgen_test] - fn test_get_update_tx_meta() { test_get_update_tx_meta_impl() } + async fn test_get_update_tx_meta() { test_get_update_tx_meta_impl().await } } diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index aceccaf9cc..d85d94e3ec 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -278,293 +278,271 @@ async fn init_nft_history_storage(chain: &Chain) -> impl NftListStorageOps + Nft storage } -pub(crate) fn test_add_get_nfts_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(nft.block_number, 28056721); - }); +pub(crate) async fn test_add_get_nfts_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); } -pub(crate) fn test_last_nft_blocks_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let nft = storage - .get_nft(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(nft.block_number, 28056721); - }); +pub(crate) async fn test_last_nft_blocks_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let nft = storage + .get_nft(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(nft.block_number, 28056721); } -pub(crate) fn test_nft_list_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let nft_list = storage - .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) - .await - .unwrap(); - assert_eq!(nft_list.nfts.len(), 1); - let nft = nft_list.nfts.get(0).unwrap(); - assert_eq!(nft.block_number, 28056721); - assert_eq!(nft_list.skipped, 1); - assert_eq!(nft_list.total, 3); - }); +pub(crate) async fn test_nft_list_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let nft_list = storage + .get_nft_list(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap())) + .await + .unwrap(); + assert_eq!(nft_list.nfts.len(), 1); + let nft = nft_list.nfts.get(0).unwrap(); + assert_eq!(nft.block_number, 28056721); + assert_eq!(nft_list.skipped, 1); + assert_eq!(nft_list.total, 3); } -pub(crate) fn test_remove_nft_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let nft_list = nft_list(); - storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let remove_rslt = storage - .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) - .await - .unwrap(); - assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); - let list_len = storage - .get_nft_list(vec![chain], true, 1, None) - .await - .unwrap() - .nfts - .len(); - assert_eq!(list_len, 2); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 28056800); - }); +pub(crate) async fn test_remove_nft_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let nft_list = nft_list(); + storage.add_nfts_to_list(&chain, nft_list, 28056726).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let remove_rslt = storage + .remove_nft_from_list(&chain, TOKEN_ADD.to_string(), token_id, 28056800) + .await + .unwrap(); + assert_eq!(remove_rslt, RemoveNftResult::NftRemoved); + let list_len = storage + .get_nft_list(vec![chain], true, 1, None) + .await + .unwrap() + .nfts + .len(); + assert_eq!(list_len, 2); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 28056800); } -pub(crate) fn test_nft_amount_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let mut nft = nft(); - storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - - nft.amount -= BigDecimal::from(1); - storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); - let amount = storage - .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "1"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919800); - - nft.amount += BigDecimal::from(1); - nft.block_number = 25919900; - storage - .update_nft_amount_and_block_number(&chain, nft.clone()) - .await - .unwrap(); - let amount = storage - .get_nft_amount(&chain, nft.token_address, nft.token_id) - .await - .unwrap() - .unwrap(); - assert_eq!(amount, "2"); - let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); - assert_eq!(last_scanned_block, 25919900); - }); +pub(crate) async fn test_nft_amount_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + + nft.amount -= BigDecimal::from(1); + storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "1"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919800); + + nft.amount += BigDecimal::from(1); + nft.block_number = 25919900; + storage + .update_nft_amount_and_block_number(&chain, nft.clone()) + .await + .unwrap(); + let amount = storage + .get_nft_amount(&chain, nft.token_address, nft.token_id) + .await + .unwrap() + .unwrap(); + assert_eq!(amount, "2"); + let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); + assert_eq!(last_scanned_block, 25919900); } -pub(crate) fn test_refresh_metadata_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_list_storage(&chain).await; - let new_symbol = "NEW_SYMBOL"; - let mut nft = nft(); - storage - .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) - .await - .unwrap(); - nft.symbol = Some(new_symbol.to_string()); - drop_mutability!(nft); - let token_add = nft.token_address.clone(); - let token_id = nft.token_id.clone(); - storage.refresh_nft_metadata(&chain, nft).await.unwrap(); - let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); - assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); - }); +pub(crate) async fn test_refresh_metadata_impl() { + let chain = Chain::Bsc; + let storage = init_nft_list_storage(&chain).await; + let new_symbol = "NEW_SYMBOL"; + let mut nft = nft(); + storage + .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) + .await + .unwrap(); + nft.symbol = Some(new_symbol.to_string()); + drop_mutability!(nft); + let token_add = nft.token_address.clone(); + let token_id = nft.token_id.clone(); + storage.refresh_nft_metadata(&chain, nft).await.unwrap(); + let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); + assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); } -pub(crate) fn test_add_get_txs_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); - let tx1 = storage - .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) - .await - .unwrap() - .get(0) - .unwrap() - .clone(); - assert_eq!(tx1.block_number, 28056721); - let tx2 = storage - .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(tx2.block_number, 28056726); - let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); - assert_eq!(tx_from.len(), 2); - }); +pub(crate) async fn test_add_get_txs_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let token_id = BigDecimal::from_str(TOKEN_ID).unwrap(); + let tx1 = storage + .get_txs_by_token_addr_id(&chain, TOKEN_ADD.to_string(), token_id) + .await + .unwrap() + .get(0) + .unwrap() + .clone(); + assert_eq!(tx1.block_number, 28056721); + let tx2 = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(tx2.block_number, 28056726); + let tx_from = storage.get_txs_from_block(&chain, 28056721).await.unwrap(); + assert_eq!(tx_from.len(), 2); } -pub(crate) fn test_last_tx_block_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) - .await - .unwrap() - .unwrap(); - assert_eq!(last_block, 28056726); - }); +pub(crate) async fn test_last_tx_block_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let last_block = NftTxHistoryStorageOps::get_last_block_number(&storage, &chain) + .await + .unwrap() + .unwrap(); + assert_eq!(last_block, 28056726); } -pub(crate) fn test_tx_history_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let tx_history = storage - .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) - .await - .unwrap(); - assert_eq!(tx_history.transfer_history.len(), 1); - let tx = tx_history.transfer_history.get(0).unwrap(); - assert_eq!(tx.block_number, 28056721); - assert_eq!(tx_history.skipped, 1); - assert_eq!(tx_history.total, 3); - }); +pub(crate) async fn test_tx_history_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let tx_history = storage + .get_tx_history(vec![chain], false, 1, Some(NonZeroUsize::new(2).unwrap()), None) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 1); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056721); + assert_eq!(tx_history.skipped, 1); + assert_eq!(tx_history.total, 3); } -pub(crate) fn test_tx_history_filters_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let filters = NftTxHistoryFilters { - receive: true, - send: false, - from_date: None, - to_date: None, - }; - - let filters1 = NftTxHistoryFilters { - receive: false, - send: false, - from_date: None, - to_date: Some(1677166110), - }; - - let filters2 = NftTxHistoryFilters { - receive: false, - send: false, - from_date: Some(1677166110), - to_date: Some(1683627417), - }; - - let tx_history = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters)) - .await - .unwrap(); - assert_eq!(tx_history.transfer_history.len(), 3); - let tx = tx_history.transfer_history.get(0).unwrap(); - assert_eq!(tx.block_number, 28056726); - - let tx_history1 = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters1)) - .await - .unwrap(); - assert_eq!(tx_history1.transfer_history.len(), 1); - let tx1 = tx_history1.transfer_history.get(0).unwrap(); - assert_eq!(tx1.block_number, 25919780); - - let tx_history2 = storage - .get_tx_history(vec![chain], true, 1, None, Some(filters2)) - .await - .unwrap(); - assert_eq!(tx_history2.transfer_history.len(), 2); - let tx_0 = tx_history2.transfer_history.get(0).unwrap(); - assert_eq!(tx_0.block_number, 28056721); - let tx_1 = tx_history2.transfer_history.get(1).unwrap(); - assert_eq!(tx_1.block_number, 25919780); - }); +pub(crate) async fn test_tx_history_filters_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let filters = NftTxHistoryFilters { + receive: true, + send: false, + from_date: None, + to_date: None, + }; + + let filters1 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: None, + to_date: Some(1677166110), + }; + + let filters2 = NftTxHistoryFilters { + receive: false, + send: false, + from_date: Some(1677166110), + to_date: Some(1683627417), + }; + + let tx_history = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters)) + .await + .unwrap(); + assert_eq!(tx_history.transfer_history.len(), 3); + let tx = tx_history.transfer_history.get(0).unwrap(); + assert_eq!(tx.block_number, 28056726); + + let tx_history1 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters1)) + .await + .unwrap(); + assert_eq!(tx_history1.transfer_history.len(), 1); + let tx1 = tx_history1.transfer_history.get(0).unwrap(); + assert_eq!(tx1.block_number, 25919780); + + let tx_history2 = storage + .get_tx_history(vec![chain], true, 1, None, Some(filters2)) + .await + .unwrap(); + assert_eq!(tx_history2.transfer_history.len(), 2); + let tx_0 = tx_history2.transfer_history.get(0).unwrap(); + assert_eq!(tx_0.block_number, 28056721); + let tx_1 = tx_history2.transfer_history.get(1).unwrap(); + assert_eq!(tx_1.block_number, 25919780); } -pub(crate) fn test_get_update_tx_meta_impl() { - futures::executor::block_on(async { - let chain = Chain::Bsc; - let storage = init_nft_history_storage(&chain).await; - let txs = nft_tx_historty(); - storage.add_txs_to_history(&chain, txs).await.unwrap(); - - let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); - assert_eq!(vec_token_add_id.len(), 2); - - let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); - let tx_meta = TxMeta { - token_address: token_add.clone(), - token_id: Default::default(), - token_uri: None, - collection_name: None, - image_url: None, - token_name: Some("Tiki box".to_string()), - }; - storage.update_txs_meta_by_token_addr_id(&chain, tx_meta).await.unwrap(); - let tx_upd = storage - .get_txs_by_token_addr_id(&chain, token_add, Default::default()) - .await - .unwrap(); - let tx_upd = tx_upd.get(0).unwrap(); - assert_eq!(tx_upd.token_name, Some("Tiki box".to_string())); - - let tx_meta = tx(); - storage.update_tx_meta_by_hash(&chain, tx_meta).await.unwrap(); - let tx_by_hash = storage - .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) - .await - .unwrap() - .unwrap(); - assert_eq!(tx_by_hash.token_name, Some("Nebula Nodes".to_string())) - }); +pub(crate) async fn test_get_update_tx_meta_impl() { + let chain = Chain::Bsc; + let storage = init_nft_history_storage(&chain).await; + let txs = nft_tx_historty(); + storage.add_txs_to_history(&chain, txs).await.unwrap(); + + let vec_token_add_id = storage.get_txs_with_empty_meta(&chain).await.unwrap(); + assert_eq!(vec_token_add_id.len(), 2); + + let token_add = "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(); + let tx_meta = TxMeta { + token_address: token_add.clone(), + token_id: Default::default(), + token_uri: None, + collection_name: None, + image_url: None, + token_name: Some("Tiki box".to_string()), + }; + storage.update_txs_meta_by_token_addr_id(&chain, tx_meta).await.unwrap(); + let tx_upd = storage + .get_txs_by_token_addr_id(&chain, token_add, Default::default()) + .await + .unwrap(); + let tx_upd = tx_upd.get(0).unwrap(); + assert_eq!(tx_upd.token_name, Some("Tiki box".to_string())); + + let tx_meta = tx(); + storage.update_tx_meta_by_hash(&chain, tx_meta).await.unwrap(); + let tx_by_hash = storage + .get_tx_by_tx_hash(&chain, TX_HASH.to_string()) + .await + .unwrap() + .unwrap(); + assert_eq!(tx_by_hash.token_name, Some("Nebula Nodes".to_string())) } From 7a821a2b0785182db5b5b2fc92bdbc2034310538 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 28 Jun 2023 14:44:54 +0700 Subject: [PATCH 103/112] test check for spam, test invalid moralis ipfs link, use is_initial_upgrade for nft wasm, add new schemes for spam check, Url for moralis ipfs --- mm2src/coins/nft.rs | 26 +++++++++++++++---- mm2src/coins/nft/nft_tests.rs | 23 +++++++++++++++- mm2src/coins/nft/storage/db_test_helpers.rs | 7 +++-- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 7 ++--- mm2src/common/common.rs | 5 ++++ 5 files changed, 57 insertions(+), 11 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 5016fc786f..14196980da 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -466,11 +466,22 @@ async fn send_request_to_uri(uri: &str) -> MmResult { Ok(response) } +/// `check_moralis_ipfs_bafy` inspects a given token URI and modifies it if certain conditions are met. +/// +/// It checks if the URI points to the Moralis IPFS domain `"ipfs.moralis.io"` and starts with a specific path prefix `"/ipfs/bafy"`. +/// If these conditions are satisfied, it modifies the URI to point to the `"ipfs.io"` domain. +/// This is due to certain "bafy"-prefixed hashes being banned on Moralis IPFS gateway due to abuse. +/// +/// If the URI does not meet these conditions or cannot be parsed, it is returned unchanged. fn check_moralis_ipfs_bafy(token_uri: Option<&str>) -> Option { token_uri.map(|uri| { - if uri.contains("https://ipfs.moralis.io") && uri.contains("bafy") { - let parts: Vec<&str> = uri.splitn(2, "/ipfs/").collect(); - format!("https://ipfs.io/ipfs/{}", parts[1]) + if let Ok(parsed_url) = Url::parse(uri) { + if parsed_url.host_str() == Some("ipfs.moralis.io") && parsed_url.path().starts_with("/ipfs/bafy") { + let parts: Vec<&str> = parsed_url.path().splitn(2, "/ipfs/").collect(); + format!("https://ipfs.io/ipfs/{}", parts[1]) + } else { + uri.to_string() + } } else { uri.to_string() } @@ -800,12 +811,17 @@ where Ok(()) } +/// `contains_disallowed_scheme` function checks if the text contains some link. +fn contains_disallowed_scheme(text: &str) -> bool { + text.contains("http://") || text.contains("https://") || text.contains("ftp://") || text.contains("file://") +} + /// `check_and_redact_if_spam` checks if the text contains any links./// /// It doesn't matter if the link is valid or not, as this is a spam check. /// If text contains some link, then it is a spam. fn check_and_redact_if_spam(text: &mut Option) -> bool { match text { - Some(s) if s.contains("http://") || s.contains("https://") => { + Some(s) if contains_disallowed_scheme(s) => { *text = Some("URL redacted for user protection".to_string()); true }, @@ -863,7 +879,7 @@ fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult, field: &str) -> bool { match metadata.get(field).and_then(|v| v.as_str()) { - Some(text) if text.contains("http://") || text.contains("https://") => { + Some(text) if contains_disallowed_scheme(text) => { metadata.insert( field.to_string(), serde_json::Value::String("URL redacted for user protection".to_string()), diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 12da8d3c9c..6d8f2e7098 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -8,7 +8,8 @@ mod native_tests { use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; - use crate::nft::{check_moralis_ipfs_bafy, send_request_to_uri}; + use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam, + send_request_to_uri}; use common::block_on; #[test] @@ -20,6 +21,26 @@ mod native_tests { assert_eq!(expected, res_uri.unwrap()); } + #[test] + fn test_invalid_moralis_ipfs_link() { + let uri = "example.com/bafy?1=ipfs.moralis.io&e=https://"; + let res_uri = check_moralis_ipfs_bafy(Some(uri)); + assert_eq!(uri, res_uri.unwrap()); + } + + #[test] + fn test_check_for_spam() { + let mut spam_text = Some("https://arweave.net".to_string()); + assert!(check_and_redact_if_spam(&mut spam_text)); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut nft = nft(); + assert!(check_nft_metadata_for_spam(&mut nft).unwrap()); + let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; + assert_eq!(meta_redacted, nft.metadata.unwrap()) + } + #[test] fn test_moralis_nft_list() { let response = block_on(send_request_to_uri(NFT_LIST_URL_TEST)).unwrap(); diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index d85d94e3ec..b36527f826 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -16,7 +16,7 @@ const TOKEN_ADD: &str = "0xfd913a305d70a60aac4faac70c739563738e1f81"; const TOKEN_ID: &str = "214300044414"; const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe"; -fn nft() -> Nft { +pub(crate) fn nft() -> Nft { Nft { chain: Chain::Bsc, token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), @@ -30,7 +30,10 @@ fn nft() -> Nft { collection_name: None, symbol: None, token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), - metadata: Some("{\"name\":\"Tiki box\"}".to_string()), + metadata: Some( + "{\"name\":\"https://arweave.net\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}" + .to_string(), + ), last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 37035879b9..eb5a2404a4 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -6,6 +6,7 @@ use crate::nft::storage::{get_offset_limit, CreateNftStorageError, NftListStorag NftTxHistoryFilters, NftTxHistoryStorageOps, RemoveNftResult}; use crate::CoinsContext; use async_trait::async_trait; +use common::is_initial_upgrade; use mm2_core::mm_ctx::MmArc; use mm2_db::indexed_db::{BeBigUint, DbTable, DbUpgrader, MultiIndex, OnUpgradeResult, SharedDb, TableSignature}; use mm2_err_handle::map_mm_error::MapMmError; @@ -585,7 +586,7 @@ impl TableSignature for NftListTable { fn table_name() -> &'static str { "nft_list_cache_table" } fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { - if let (0, 1) = (old_version, new_version) { + if is_initial_upgrade(old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; table.create_multi_index( Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, @@ -650,7 +651,7 @@ impl TableSignature for NftTxHistoryTable { fn table_name() -> &'static str { "nft_tx_history_cache_table" } fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { - if let (0, 1) = (old_version, new_version) { + if is_initial_upgrade(old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; table.create_multi_index( Self::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX, @@ -676,7 +677,7 @@ impl TableSignature for LastScannedBlockTable { fn table_name() -> &'static str { "last_scanned_block_table" } fn on_upgrade_needed(upgrader: &DbUpgrader, old_version: u32, new_version: u32) -> OnUpgradeResult<()> { - if let (0, 1) = (old_version, new_version) { + if is_initial_upgrade(old_version, new_version) { let table = upgrader.create_table(Self::table_name())?; table.create_index("chain", true)?; } diff --git a/mm2src/common/common.rs b/mm2src/common/common.rs index d43bb254f8..653ad11353 100644 --- a/mm2src/common/common.rs +++ b/mm2src/common/common.rs @@ -1040,3 +1040,8 @@ pub fn parse_rfc3339_to_timestamp(date_str: &str) -> Result = date_str.parse()?; Ok(date.timestamp().try_into()?) } + +/// `is_initial_upgrade` function checks if the database is being upgraded from version 0 to 1. +/// This function returns a boolean indicating whether the database is being upgraded from version 0 to 1. +#[cfg(target_arch = "wasm32")] +pub fn is_initial_upgrade(old_version: u32, new_version: u32) -> bool { old_version == 0 && new_version == 1 } From 167bde2b214c76467131843b77cd82d242148656 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 29 Jun 2023 12:00:53 +0700 Subject: [PATCH 104/112] rename `merge_from` to `try_to_fill_missing_fields_from` --- mm2src/coins/nft.rs | 2 +- mm2src/coins/nft/nft_structs.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 14196980da..9e7adde2cd 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -499,7 +499,7 @@ async fn get_uri_meta(token_uri: Option<&str>, metadata: Option<&str>) -> UriMet } if let Some(metadata) = metadata { if let Ok(meta_from_meta) = serde_json::from_str::(metadata) { - uri_meta.merge_from(meta_from_meta) + uri_meta.try_to_fill_missing_fields_from(meta_from_meta) } } uri_meta.image_url = check_moralis_ipfs_bafy(uri_meta.image_url.as_deref()); diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 4436316451..0630c54641 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -156,11 +156,11 @@ pub(crate) struct UriMeta { } impl UriMeta { - /// `merge_from` function doesnt change `raw_image_url` field. + /// `try_to_fill_missing_fields_from` function doesnt change `raw_image_url` field. /// It tries to update `image_url` field instead, if it is None. /// As `image` is the original name of `raw_image_url` field in data from `token_uri` or `metadata`, /// try to find **Some()** in this field first. - pub(crate) fn merge_from(&mut self, other: UriMeta) { + pub(crate) fn try_to_fill_missing_fields_from(&mut self, other: UriMeta) { if self.image_url.is_none() { self.image_url = other.raw_image_url.or(other.image_url); } From 1746503abd52c0e0d24f4d4326df7cb38547596a Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 29 Jun 2023 17:07:15 +0700 Subject: [PATCH 105/112] add my_address != tx.from_address check when receive erc721, make transaction_type, verified, token_hash optional --- mm2src/coins/nft.rs | 64 +++++++++++++++------ mm2src/coins/nft/nft_errors.rs | 5 ++ mm2src/coins/nft/nft_structs.rs | 12 ++-- mm2src/coins/nft/storage/db_test_helpers.rs | 24 ++++---- 4 files changed, 69 insertions(+), 36 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 9e7adde2cd..96f550f34f 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -590,24 +590,52 @@ async fn handle_receive_erc721( url: &Url, my_address: &str, ) -> MmResult<(), UpdateNftError> { - let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; - // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later - // than History by Wallet update - nft.owner_of = my_address.to_string(); - nft.block_number = tx.block_number; - drop_mutability!(nft); - storage - .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number) - .await?; - let tx_meta = TxMeta { - token_address: nft.token_address, - token_id: nft.token_id, - token_uri: nft.token_uri, - collection_name: nft.collection_name, - image_url: nft.uri_meta.image_url, - token_name: nft.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + // If token isn't in NFT LIST table then add nft to the table. + if let Some(mut nft_db) = storage + .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .await? + { + // An error is raised if user tries to receive an identical ERC-721 token they already own + // and if owner address != from address + if my_address != tx.from_address { + return MmError::err(UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { + tx_hash: tx.transaction_hash, + }); + } + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage + .update_nft_amount_and_block_number(chain, nft_db.clone()) + .await?; + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + token_uri: nft_db.token_uri, + collection_name: nft_db.collection_name, + image_url: nft_db.uri_meta.image_url, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + } else { + let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; + // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later + // than History by Wallet update + nft.owner_of = my_address.to_string(); + nft.block_number = tx.block_number; + drop_mutability!(nft); + storage + .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number) + .await?; + let tx_meta = TxMeta { + token_address: nft.token_address, + token_id: nft.token_id, + token_uri: nft.token_uri, + collection_name: nft.collection_name, + image_url: nft.uri_meta.image_url, + token_name: nft.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; + } Ok(()) } diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index 4889d74bba..e00e1c9bd0 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -156,6 +156,10 @@ pub enum UpdateNftError { LastScannedBlockNotFound { last_nft_block: String, }, + #[display(fmt = "Attempt to receive duplicate ERC721 token in transaction hash: {}", tx_hash)] + AttemptToReceiveAlreadyOwnedErc721 { + tx_hash: String, + }, } impl From for UpdateNftError { @@ -189,6 +193,7 @@ impl HttpStatusCode for UpdateNftError { | UpdateNftError::InsufficientAmountInCache { .. } | UpdateNftError::InvalidBlockOrder { .. } | UpdateNftError::LastScannedBlockNotFound { .. } => StatusCode::INTERNAL_SERVER_ERROR, + UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 0630c54641..ecd2dec91c 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -192,7 +192,7 @@ pub struct Nft { pub(crate) token_id: BigDecimal, pub(crate) amount: BigDecimal, pub(crate) owner_of: String, - pub(crate) token_hash: String, + pub(crate) token_hash: Option, pub(crate) block_number_minted: u64, pub(crate) block_number: u64, pub(crate) contract_type: ContractType, @@ -215,7 +215,7 @@ pub(crate) struct NftWrapper { pub(crate) token_id: SerdeStringWrap, pub(crate) amount: SerdeStringWrap, pub(crate) owner_of: String, - pub(crate) token_hash: String, + pub(crate) token_hash: Option, pub(crate) block_number_minted: SerdeStringWrap, pub(crate) block_number: SerdeStringWrap, pub(crate) contract_type: Option>, @@ -374,7 +374,7 @@ pub struct NftTransferHistory { pub(crate) log_index: u64, pub(crate) value: BigDecimal, pub(crate) contract_type: ContractType, - pub(crate) transaction_type: String, + pub(crate) transaction_type: Option, pub(crate) token_address: String, pub(crate) token_id: BigDecimal, pub(crate) token_uri: Option, @@ -385,7 +385,7 @@ pub struct NftTransferHistory { pub(crate) to_address: String, pub(crate) status: TransferStatus, pub(crate) amount: BigDecimal, - pub(crate) verified: u64, + pub(crate) verified: Option, pub(crate) operator: Option, pub(crate) possible_spam: bool, } @@ -401,13 +401,13 @@ pub(crate) struct NftTransferHistoryWrapper { pub(crate) log_index: u64, pub(crate) value: SerdeStringWrap, pub(crate) contract_type: Option>, - pub(crate) transaction_type: String, + pub(crate) transaction_type: Option, pub(crate) token_address: String, pub(crate) token_id: SerdeStringWrap, pub(crate) from_address: String, pub(crate) to_address: String, pub(crate) amount: SerdeStringWrap, - pub(crate) verified: u64, + pub(crate) verified: Option, pub(crate) operator: Option, #[serde(default)] pub(crate) possible_spam: bool, diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index b36527f826..51a017a135 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -23,7 +23,7 @@ pub(crate) fn nft() -> Nft { token_id: Default::default(), amount: BigDecimal::from_str("2").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "b34ddf294013d20a6d70691027625839".to_string(), + token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), block_number_minted: 25465916, block_number: 25919780, contract_type: ContractType::Erc1155, @@ -62,7 +62,7 @@ fn tx() -> NftTransferHistory { log_index: 495, value: Default::default(), contract_type: ContractType::Erc721, - transaction_type: "Single".to_string(), + transaction_type: Some("Single".to_string()), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300047252").unwrap(), token_uri: None, @@ -73,7 +73,7 @@ fn tx() -> NftTransferHistory { to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, amount: BigDecimal::from_str("1").unwrap(), - verified: 1, + verified: Some(1), operator: None, possible_spam: false, } @@ -86,7 +86,7 @@ fn nft_list() -> Vec { token_id: Default::default(), amount: BigDecimal::from_str("2").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "b34ddf294013d20a6d70691027625839".to_string(), + token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), block_number_minted: 25465916, block_number: 25919780, contract_type: ContractType::Erc1155, @@ -116,7 +116,7 @@ fn nft_list() -> Vec { token_id: BigDecimal::from_str("214300047252").unwrap(), amount: BigDecimal::from_str("1").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "c5d1cfd75a0535b0ec750c0156e6ddfe".to_string(), + token_hash: Some("c5d1cfd75a0535b0ec750c0156e6ddfe".to_string()), block_number_minted: 25721963, block_number: 28056726, contract_type: ContractType::Erc721, @@ -151,7 +151,7 @@ fn nft_list() -> Vec { token_id: BigDecimal::from_str("214300044414").unwrap(), amount: BigDecimal::from_str("1").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: "125f8f4e952e107c257960000b4b250e".to_string(), + token_hash: Some("125f8f4e952e107c257960000b4b250e".to_string()), block_number_minted: 25810308, block_number: 28056721, contract_type: ContractType::Erc721, @@ -193,7 +193,7 @@ fn nft_tx_historty() -> Vec { log_index: 139, value: Default::default(), contract_type: ContractType::Erc1155, - transaction_type: "Single".to_string(), + transaction_type: Some("Single".to_string()), token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), token_id: Default::default(), token_uri: None, @@ -204,7 +204,7 @@ fn nft_tx_historty() -> Vec { to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, amount: BigDecimal::from_str("1").unwrap(), - verified: 1, + verified: Some(1), operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), possible_spam: false, }; @@ -219,7 +219,7 @@ fn nft_tx_historty() -> Vec { log_index: 495, value: Default::default(), contract_type: ContractType::Erc721, - transaction_type: "Single".to_string(), + transaction_type: Some("Single".to_string()), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300047252").unwrap(), token_uri: None, @@ -230,7 +230,7 @@ fn nft_tx_historty() -> Vec { to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, amount: BigDecimal::from_str("1").unwrap(), - verified: 1, + verified: Some(1), operator: None, possible_spam: false, }; @@ -245,7 +245,7 @@ fn nft_tx_historty() -> Vec { log_index: 201, value: Default::default(), contract_type: ContractType::Erc721, - transaction_type: "Single".to_string(), + transaction_type: Some("Single".to_string()), token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), token_id: BigDecimal::from_str("214300044414").unwrap(), token_uri: None, @@ -256,7 +256,7 @@ fn nft_tx_historty() -> Vec { to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, amount: BigDecimal::from_str("1").unwrap(), - verified: 1, + verified: Some(1), operator: None, possible_spam: false, }; From 755ebc64e44399ee5b270d3f9be44d84639b29d1 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 29 Jun 2023 19:48:48 +0700 Subject: [PATCH 106/112] make more fields optional --- mm2src/coins/nft.rs | 6 ++-- mm2src/coins/nft/nft_structs.rs | 20 ++++++------- mm2src/coins/nft/nft_tests.rs | 4 +-- mm2src/coins/nft/storage/db_test_helpers.rs | 32 ++++++++++----------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 96f550f34f..f670fbc712 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -237,7 +237,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult amount: nft_wrapper.amount.0, owner_of: nft_wrapper.owner_of, token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, + block_number_minted: nft_wrapper.block_number_minted.map(|v| v.0), block_number: *nft_wrapper.block_number, contract_type, collection_name: nft_wrapper.name, @@ -319,7 +319,7 @@ async fn get_moralis_nft_transfers( transaction_hash: transfer_wrapper.transaction_hash, transaction_index: transfer_wrapper.transaction_index, log_index: transfer_wrapper.log_index, - value: transfer_wrapper.value.0, + value: transfer_wrapper.value.map(|v| v.0), contract_type, transaction_type: transfer_wrapper.transaction_type, token_address: transfer_wrapper.token_address, @@ -389,7 +389,7 @@ async fn get_moralis_metadata( amount: nft_wrapper.amount.0, owner_of: nft_wrapper.owner_of, token_hash: nft_wrapper.token_hash, - block_number_minted: *nft_wrapper.block_number_minted, + block_number_minted: nft_wrapper.block_number_minted.map(|v| v.0), block_number: *nft_wrapper.block_number, contract_type, collection_name: nft_wrapper.name, diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index ecd2dec91c..6db2ed8671 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -193,7 +193,7 @@ pub struct Nft { pub(crate) amount: BigDecimal, pub(crate) owner_of: String, pub(crate) token_hash: Option, - pub(crate) block_number_minted: u64, + pub(crate) block_number_minted: Option, pub(crate) block_number: u64, pub(crate) contract_type: ContractType, pub(crate) collection_name: Option, @@ -216,7 +216,7 @@ pub(crate) struct NftWrapper { pub(crate) amount: SerdeStringWrap, pub(crate) owner_of: String, pub(crate) token_hash: Option, - pub(crate) block_number_minted: SerdeStringWrap, + pub(crate) block_number_minted: Option>, pub(crate) block_number: SerdeStringWrap, pub(crate) contract_type: Option>, pub(crate) name: Option, @@ -367,12 +367,12 @@ pub struct NftTransferHistory { pub(crate) chain: Chain, pub(crate) block_number: u64, pub(crate) block_timestamp: u64, - pub(crate) block_hash: String, + pub(crate) block_hash: Option, /// Transaction hash in hexadecimal format pub(crate) transaction_hash: String, - pub(crate) transaction_index: u64, - pub(crate) log_index: u64, - pub(crate) value: BigDecimal, + pub(crate) transaction_index: Option, + pub(crate) log_index: Option, + pub(crate) value: Option, pub(crate) contract_type: ContractType, pub(crate) transaction_type: Option, pub(crate) token_address: String, @@ -394,12 +394,12 @@ pub struct NftTransferHistory { pub(crate) struct NftTransferHistoryWrapper { pub(crate) block_number: SerdeStringWrap, pub(crate) block_timestamp: String, - pub(crate) block_hash: String, + pub(crate) block_hash: Option, /// Transaction hash in hexadecimal format pub(crate) transaction_hash: String, - pub(crate) transaction_index: u64, - pub(crate) log_index: u64, - pub(crate) value: SerdeStringWrap, + pub(crate) transaction_index: Option, + pub(crate) log_index: Option, + pub(crate) value: Option>, pub(crate) contract_type: Option>, pub(crate) transaction_type: Option, pub(crate) token_address: String, diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 6d8f2e7098..97704adee3 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -65,7 +65,7 @@ mod native_tests { fn test_moralis_nft_metadata() { let response = block_on(send_request_to_uri(NFT_METADATA_URL_TEST)).unwrap(); let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); - assert_eq!(41237364, *nft_wrapper.block_number_minted); + assert_eq!(41237364, *nft_wrapper.block_number_minted.unwrap()); let token_uri = nft_wrapper.token_uri.unwrap(); let uri_response = block_on(send_request_to_uri(token_uri.as_str())).unwrap(); serde_json::from_str::(&uri_response.to_string()).unwrap(); @@ -139,7 +139,7 @@ mod wasm_tests { async fn test_moralis_nft_metadata() { let response = send_request_to_uri(NFT_METADATA_URL_TEST).await.unwrap(); let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); - assert_eq!(41237364, *nft_wrapper.block_number_minted); + assert_eq!(41237364, *nft_wrapper.block_number_minted.unwrap()); } #[wasm_bindgen_test] diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index 51a017a135..fcb00d4893 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -24,7 +24,7 @@ pub(crate) fn nft() -> Nft { amount: BigDecimal::from_str("2").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), - block_number_minted: 25465916, + block_number_minted: Some(25465916), block_number: 25919780, contract_type: ContractType::Erc1155, collection_name: None, @@ -56,10 +56,10 @@ fn tx() -> NftTransferHistory { chain: Chain::Bsc, block_number: 28056726, block_timestamp: 1683627432, - block_hash: "0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string(), + block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), - transaction_index: 198, - log_index: 495, + transaction_index: Some(198), + log_index: Some(495), value: Default::default(), contract_type: ContractType::Erc721, transaction_type: Some("Single".to_string()), @@ -87,7 +87,7 @@ fn nft_list() -> Vec { amount: BigDecimal::from_str("2").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), - block_number_minted: 25465916, + block_number_minted: Some(25465916), block_number: 25919780, contract_type: ContractType::Erc1155, collection_name: None, @@ -117,7 +117,7 @@ fn nft_list() -> Vec { amount: BigDecimal::from_str("1").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), token_hash: Some("c5d1cfd75a0535b0ec750c0156e6ddfe".to_string()), - block_number_minted: 25721963, + block_number_minted: Some(25721963), block_number: 28056726, contract_type: ContractType::Erc721, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), @@ -152,7 +152,7 @@ fn nft_list() -> Vec { amount: BigDecimal::from_str("1").unwrap(), owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), token_hash: Some("125f8f4e952e107c257960000b4b250e".to_string()), - block_number_minted: 25810308, + block_number_minted: Some(25810308), block_number: 28056721, contract_type: ContractType::Erc721, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), @@ -187,10 +187,10 @@ fn nft_tx_historty() -> Vec { chain: Chain::Bsc, block_number: 25919780, block_timestamp: 1677166110, - block_hash: "0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string(), + block_hash: Some("0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string()), transaction_hash: "0x9c16b962f63eead1c5d2355cc9037dde178b14b53043c57eb40c27964d22ae6a".to_string(), - transaction_index: 57, - log_index: 139, + transaction_index: Some(57), + log_index: Some(139), value: Default::default(), contract_type: ContractType::Erc1155, transaction_type: Some("Single".to_string()), @@ -213,10 +213,10 @@ fn nft_tx_historty() -> Vec { chain: Chain::Bsc, block_number: 28056726, block_timestamp: 1683627432, - block_hash: "0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string(), + block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), - transaction_index: 198, - log_index: 495, + transaction_index: Some(198), + log_index: Some(495), value: Default::default(), contract_type: ContractType::Erc721, transaction_type: Some("Single".to_string()), @@ -239,10 +239,10 @@ fn nft_tx_historty() -> Vec { chain: Chain::Bsc, block_number: 28056721, block_timestamp: 1683627417, - block_hash: "0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string(), + block_hash: Some("0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string()), transaction_hash: "0x981bad702cc6e088f0e9b5e7287ff7a3487b8d269103cee3b9e5803141f63f91".to_string(), - transaction_index: 83, - log_index: 201, + transaction_index: Some(83), + log_index: Some(201), value: Default::default(), contract_type: ContractType::Erc721, transaction_type: Some("Single".to_string()), From 8471fbce77351c14edbf4556247f28e2cd3e9a25 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 30 Jun 2023 12:50:42 +0700 Subject: [PATCH 107/112] use match in `handle_receive_erc721`, handle Erc721 error --- mm2src/coins/nft.rs | 82 ++++++++++++++++------------------ mm2src/coins/nft/nft_errors.rs | 4 +- 2 files changed, 41 insertions(+), 45 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index f670fbc712..1f0e62edd6 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -590,52 +590,48 @@ async fn handle_receive_erc721( url: &Url, my_address: &str, ) -> MmResult<(), UpdateNftError> { - // If token isn't in NFT LIST table then add nft to the table. - if let Some(mut nft_db) = storage + let nft = match storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await? { - // An error is raised if user tries to receive an identical ERC-721 token they already own - // and if owner address != from address - if my_address != tx.from_address { - return MmError::err(UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { - tx_hash: tx.transaction_hash, - }); - } - nft_db.block_number = tx.block_number; - drop_mutability!(nft_db); - storage - .update_nft_amount_and_block_number(chain, nft_db.clone()) - .await?; - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - } else { - let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; - // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later - // than History by Wallet update - nft.owner_of = my_address.to_string(); - nft.block_number = tx.block_number; - drop_mutability!(nft); - storage - .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number) - .await?; - let tx_meta = TxMeta { - token_address: nft.token_address, - token_id: nft.token_id, - token_uri: nft.token_uri, - collection_name: nft.collection_name, - image_url: nft.uri_meta.image_url, - token_name: nft.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - } + Some(mut nft_db) => { + // An error is raised if user tries to receive an identical ERC-721 token they already own + // and if owner address != from address + if my_address != tx.from_address { + return MmError::err(UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { + tx_hash: tx.transaction_hash, + }); + } + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage + .update_nft_amount_and_block_number(chain, nft_db.clone()) + .await?; + nft_db + }, + // If token isn't in NFT LIST table then add nft to the table. + None => { + let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; + // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later + // than History by Wallet update + nft.owner_of = my_address.to_string(); + nft.block_number = tx.block_number; + drop_mutability!(nft); + storage + .add_nfts_to_list(chain, vec![nft.clone()], tx.block_number) + .await?; + nft + }, + }; + let tx_meta = TxMeta { + token_address: nft.token_address, + token_id: nft.token_id, + token_uri: nft.token_uri, + collection_name: nft.collection_name, + image_url: nft.uri_meta.image_url, + token_name: nft.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; Ok(()) } diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index e00e1c9bd0..c2f074d02b 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -192,8 +192,8 @@ impl HttpStatusCode for UpdateNftError { | UpdateNftError::TokenNotFoundInWallet { .. } | UpdateNftError::InsufficientAmountInCache { .. } | UpdateNftError::InvalidBlockOrder { .. } - | UpdateNftError::LastScannedBlockNotFound { .. } => StatusCode::INTERNAL_SERVER_ERROR, - UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { .. } => StatusCode::INTERNAL_SERVER_ERROR, + | UpdateNftError::LastScannedBlockNotFound { .. } + | UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { .. } => StatusCode::INTERNAL_SERVER_ERROR, } } } From cf6ff1ad1450aee05f9d9ce2926d0c38c47eb551 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 30 Jun 2023 15:41:32 +0700 Subject: [PATCH 108/112] use match in `handle_receive_erc1155`, in `handle_send_erc1155` move `TxMeta` update to the end, make from_block=1 in `get_moralis_nft_transfers` --- mm2src/coins/nft.rs | 138 ++++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 76 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 1f0e62edd6..69341dd8a0 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -286,7 +286,7 @@ async fn get_moralis_nft_transfers( .push("transfers"); let from_block = match from_block { Some(block) => block.to_string(), - None => "0".into(), + None => "1".into(), }; uri_without_cursor .query_pairs_mut() @@ -649,34 +649,15 @@ async fn handle_send_erc1155( })?; match nft_db.amount.cmp(&tx.amount) { Ordering::Equal => { - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; storage .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) .await?; }, Ordering::Greater => { nft_db.amount -= tx.amount; - drop_mutability!(nft_db); storage .update_nft_amount(chain, nft_db.clone(), tx.block_number) .await?; - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; }, Ordering::Less => { return MmError::err(UpdateNftError::InsufficientAmountInCache { @@ -685,6 +666,15 @@ async fn handle_send_erc1155( }); }, } + let tx_meta = TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + token_uri: nft_db.token_uri, + collection_name: nft_db.collection_name, + image_url: nft_db.uri_meta.image_url, + token_name: nft_db.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; Ok(()) } @@ -695,65 +685,61 @@ async fn handle_receive_erc1155( url: &Url, my_address: &str, ) -> MmResult<(), UpdateNftError> { - // If token isn't in NFT LIST table then add nft to the table. - if let Some(mut nft_db) = storage + let nft = match storage .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) .await? { - // if owner address == from address, then owner sent tokens to themself, - // which means that the amount will not change. - if my_address != tx.from_address { - nft_db.amount += tx.amount; - } - nft_db.block_number = tx.block_number; - drop_mutability!(nft_db); - storage - .update_nft_amount_and_block_number(chain, nft_db.clone()) - .await?; - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - } else { - let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; - let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.metadata.as_deref()).await; - let nft = Nft { - chain: *chain, - token_address: moralis_meta.token_address, - token_id: moralis_meta.token_id, - amount: tx.amount, - owner_of: my_address.to_string(), - token_hash: moralis_meta.token_hash, - block_number_minted: moralis_meta.block_number_minted, - block_number: tx.block_number, - contract_type: moralis_meta.contract_type, - collection_name: moralis_meta.collection_name, - symbol: moralis_meta.symbol, - token_uri, - metadata: moralis_meta.metadata, - last_token_uri_sync: moralis_meta.last_token_uri_sync, - last_metadata_sync: moralis_meta.last_metadata_sync, - minter_address: moralis_meta.minter_address, - possible_spam: moralis_meta.possible_spam, - uri_meta, - }; - storage.add_nfts_to_list(chain, [nft.clone()], tx.block_number).await?; - let tx_meta = TxMeta { - token_address: nft.token_address, - token_id: nft.token_id, - token_uri: nft.token_uri, - collection_name: nft.collection_name, - image_url: nft.uri_meta.image_url, - token_name: nft.uri_meta.token_name, - }; - storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; - } + Some(mut nft_db) => { + // if owner address == from address, then owner sent tokens to themself, + // which means that the amount will not change. + if my_address != tx.from_address { + nft_db.amount += tx.amount; + } + nft_db.block_number = tx.block_number; + drop_mutability!(nft_db); + storage + .update_nft_amount_and_block_number(chain, nft_db.clone()) + .await?; + nft_db + }, + // If token isn't in NFT LIST table then add nft to the table. + None => { + let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; + let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.metadata.as_deref()).await; + let nft = Nft { + chain: *chain, + token_address: moralis_meta.token_address, + token_id: moralis_meta.token_id, + amount: tx.amount, + owner_of: my_address.to_string(), + token_hash: moralis_meta.token_hash, + block_number_minted: moralis_meta.block_number_minted, + block_number: tx.block_number, + contract_type: moralis_meta.contract_type, + collection_name: moralis_meta.collection_name, + symbol: moralis_meta.symbol, + token_uri, + metadata: moralis_meta.metadata, + last_token_uri_sync: moralis_meta.last_token_uri_sync, + last_metadata_sync: moralis_meta.last_metadata_sync, + minter_address: moralis_meta.minter_address, + possible_spam: moralis_meta.possible_spam, + uri_meta, + }; + storage.add_nfts_to_list(chain, [nft.clone()], tx.block_number).await?; + nft + }, + }; + let tx_meta = TxMeta { + token_address: nft.token_address, + token_id: nft.token_id, + token_uri: nft.token_uri, + collection_name: nft.collection_name, + image_url: nft.uri_meta.image_url, + token_name: nft.uri_meta.token_name, + }; + storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; Ok(()) } From 478b178adc8bfc2d7d2756d6fd7c1d50f05a0313 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 4 Jul 2023 16:39:53 +0700 Subject: [PATCH 109/112] use map in `check_nft_metadata_for_spam`, impl From for `TxMeta` --- mm2src/coins/nft.rs | 78 +++++++-------------------------- mm2src/coins/nft/nft_structs.rs | 13 ++++++ 2 files changed, 29 insertions(+), 62 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 69341dd8a0..6f7f6573a2 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -187,14 +187,7 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu storage .refresh_nft_metadata(&moralis_meta.chain, nft_db.clone()) .await?; - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft_db.clone()); storage.update_txs_meta_by_token_addr_id(&nft_db.chain, tx_meta).await?; Ok(()) } @@ -568,14 +561,7 @@ async fn handle_send_erc721( token_address: tx.token_address.clone(), token_id: tx.token_id.to_string(), })?; - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft_db); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; storage .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) @@ -623,14 +609,7 @@ async fn handle_receive_erc721( nft }, }; - let tx_meta = TxMeta { - token_address: nft.token_address, - token_id: nft.token_id, - token_uri: nft.token_uri, - collection_name: nft.collection_name, - image_url: nft.uri_meta.image_url, - token_name: nft.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; Ok(()) } @@ -666,14 +645,7 @@ async fn handle_send_erc1155( }); }, } - let tx_meta = TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, - image_url: nft_db.uri_meta.image_url, - token_name: nft_db.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft_db); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; Ok(()) } @@ -731,14 +703,7 @@ async fn handle_receive_erc1155( nft }, }; - let tx_meta = TxMeta { - token_address: nft.token_address, - token_id: nft.token_id, - token_uri: nft.token_uri, - collection_name: nft.collection_name, - image_url: nft.uri_meta.image_url, - token_name: nft.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; Ok(()) } @@ -787,14 +752,7 @@ where T: NftListStorageOps + NftTxHistoryStorageOps, { for nft in nfts.into_iter() { - let tx_meta = TxMeta { - token_address: nft.token_address, - token_id: nft.token_id, - token_uri: nft.token_uri, - collection_name: nft.collection_name, - image_url: nft.uri_meta.image_url, - token_name: nft.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } Ok(()) @@ -808,14 +766,7 @@ where let nft_token_addr_id = storage.get_txs_with_empty_meta(chain).await?; for addr_id_pair in nft_token_addr_id.into_iter() { let nft_meta = get_moralis_metadata(addr_id_pair.token_address, addr_id_pair.token_id, chain, url).await?; - let tx_meta = TxMeta { - token_address: nft_meta.token_address, - token_id: nft_meta.token_id, - token_uri: nft_meta.token_uri, - collection_name: nft_meta.collection_name, - image_url: nft_meta.uri_meta.image_url, - token_name: nft_meta.uri_meta.token_name, - }; + let tx_meta = TxMeta::from(nft_meta); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; } Ok(()) @@ -838,6 +789,7 @@ fn check_and_redact_if_spam(text: &mut Option) -> bool { _ => false, } } + /// `protect_from_history_spam` function checks and redact spam in `NftTransferHistory`. /// /// `collection_name` and `token_name` in `NftTransferHistory` shouldn't contain any links, @@ -870,12 +822,14 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { /// /// **note:** `token_name` is usually called `name` in `metadata`. fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { - if let Some(metadata_str) = &nft.metadata { - if let Ok(mut metadata) = serde_json::from_str::>(metadata_str) { - if check_spam_and_redact_metadata_field(&mut metadata, "name") { - nft.metadata = Some(serde_json::to_string(&metadata)?); - return Ok(true); - } + if let Some(Ok(mut metadata)) = nft + .metadata + .as_ref() + .map(|t| serde_json::from_str::>(t)) + { + if check_spam_and_redact_metadata_field(&mut metadata, "name") { + nft.metadata = Some(serde_json::to_string(&metadata)?); + return Ok(true); } } Ok(false) diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 6db2ed8671..3f50749c5b 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -451,3 +451,16 @@ pub struct TxMeta { pub(crate) image_url: Option, pub(crate) token_name: Option, } + +impl From for TxMeta { + fn from(nft_db: Nft) -> Self { + TxMeta { + token_address: nft_db.token_address, + token_id: nft_db.token_id, + token_uri: nft_db.token_uri, + collection_name: nft_db.collection_name, + image_url: nft_db.uri_meta.image_url, + token_name: nft_db.uri_meta.token_name, + } + } +} From 345cf5f0dafcb588a5216f991c869764ad19a30e Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 5 Jul 2023 20:35:02 +0700 Subject: [PATCH 110/112] use common structures, combine moralis requests into one test --- mm2src/coins/nft.rs | 238 ++++++++------- mm2src/coins/nft/nft_structs.rs | 101 +++---- mm2src/coins/nft/nft_tests.rs | 68 ++--- mm2src/coins/nft/storage/db_test_helpers.rs | 278 ++++++++++-------- mm2src/coins/nft/storage/sql_storage.rs | 30 +- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 32 +- 6 files changed, 380 insertions(+), 367 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 6f7f6573a2..4371c13571 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -10,12 +10,12 @@ pub(crate) mod storage; use crate::{get_my_address, MyAddressReq, WithdrawError}; use nft_errors::{GetInfoFromUriError, GetNftInfoError, UpdateNftError}; -use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftList, NftListReq, NftMetadataReq, NftTransferHistory, - NftTransferHistoryWrapper, NftTransfersReq, NftWrapper, NftsTransferHistoryList, +use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftList, NftListReq, NftMetadataReq, + NftTransferHistory, NftTransfersReq, NftTxHistoryFromMoralis, NftsTransferHistoryList, TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; -use crate::nft::nft_structs::{RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; +use crate::nft::nft_structs::{NftCommon, NftTransferCommon, RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use http::header::ACCEPT; @@ -173,15 +173,15 @@ pub async fn refresh_nft_metadata(ctx: MmArc, req: RefreshMetadataReq) -> MmResu protect_from_spam: false, }; let mut nft_db = get_nft_metadata(ctx, req).await?; - let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.metadata.as_deref()).await; - nft_db.collection_name = moralis_meta.collection_name; - nft_db.symbol = moralis_meta.symbol; - nft_db.token_uri = token_uri; - nft_db.metadata = moralis_meta.metadata; - nft_db.last_token_uri_sync = moralis_meta.last_token_uri_sync; - nft_db.last_metadata_sync = moralis_meta.last_metadata_sync; - nft_db.possible_spam = moralis_meta.possible_spam; + let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.common.metadata.as_deref()).await; + nft_db.common.collection_name = moralis_meta.common.collection_name; + nft_db.common.symbol = moralis_meta.common.symbol; + nft_db.common.token_uri = token_uri; + nft_db.common.metadata = moralis_meta.common.metadata; + nft_db.common.last_token_uri_sync = moralis_meta.common.last_token_uri_sync; + nft_db.common.last_metadata_sync = moralis_meta.common.last_metadata_sync; + nft_db.common.possible_spam = moralis_meta.common.possible_spam; nft_db.uri_meta = uri_meta; drop_mutability!(nft_db); storage @@ -216,31 +216,33 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult let response = send_request_to_uri(uri.as_str()).await?; if let Some(nfts_list) = response["result"].as_array() { for nft_json in nfts_list { - let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string())?; - let contract_type = match nft_wrapper.contract_type { - Some(contract_type) => contract_type.0, + let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string())?; + let contract_type = match nft_moralis.contract_type { + Some(contract_type) => contract_type, None => continue, }; - let token_uri = check_moralis_ipfs_bafy(nft_wrapper.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), nft_wrapper.metadata.as_deref()).await; + let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; let nft = Nft { + common: NftCommon { + token_address: nft_moralis.common.token_address, + token_id: nft_moralis.common.token_id, + amount: nft_moralis.common.amount, + owner_of: nft_moralis.common.owner_of, + token_hash: nft_moralis.common.token_hash, + collection_name: nft_moralis.common.collection_name, + symbol: nft_moralis.common.symbol, + token_uri, + metadata: nft_moralis.common.metadata, + last_token_uri_sync: nft_moralis.common.last_token_uri_sync, + last_metadata_sync: nft_moralis.common.last_metadata_sync, + minter_address: nft_moralis.common.minter_address, + possible_spam: nft_moralis.common.possible_spam, + }, chain: *chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: nft_wrapper.block_number_minted.map(|v| v.0), - block_number: *nft_wrapper.block_number, + block_number_minted: nft_moralis.block_number_minted.map(|v| v.0), + block_number: *nft_moralis.block_number, contract_type, - collection_name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, uri_meta, }; // collect NFTs from the page @@ -297,37 +299,39 @@ async fn get_moralis_nft_transfers( let response = send_request_to_uri(uri.as_str()).await?; if let Some(transfer_list) = response["result"].as_array() { for transfer in transfer_list { - let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&transfer.to_string())?; - let contract_type = match transfer_wrapper.contract_type { - Some(contract_type) => contract_type.0, + let transfer_moralis: NftTxHistoryFromMoralis = serde_json::from_str(&transfer.to_string())?; + let contract_type = match transfer_moralis.contract_type { + Some(contract_type) => contract_type, None => continue, }; - let status = get_tx_status(&wallet_address, &transfer_wrapper.to_address); - let block_timestamp = parse_rfc3339_to_timestamp(&transfer_wrapper.block_timestamp)?; + let status = get_tx_status(&wallet_address, &transfer_moralis.common.to_address); + let block_timestamp = parse_rfc3339_to_timestamp(&transfer_moralis.block_timestamp)?; let transfer_history = NftTransferHistory { + common: NftTransferCommon { + block_hash: transfer_moralis.common.block_hash, + transaction_hash: transfer_moralis.common.transaction_hash, + transaction_index: transfer_moralis.common.transaction_index, + log_index: transfer_moralis.common.log_index, + value: transfer_moralis.common.value, + transaction_type: transfer_moralis.common.transaction_type, + token_address: transfer_moralis.common.token_address, + token_id: transfer_moralis.common.token_id, + from_address: transfer_moralis.common.from_address, + to_address: transfer_moralis.common.to_address, + amount: transfer_moralis.common.amount, + verified: transfer_moralis.common.verified, + operator: transfer_moralis.common.operator, + possible_spam: transfer_moralis.common.possible_spam, + }, chain: *chain, - block_number: *transfer_wrapper.block_number, + block_number: *transfer_moralis.block_number, block_timestamp, - block_hash: transfer_wrapper.block_hash, - transaction_hash: transfer_wrapper.transaction_hash, - transaction_index: transfer_wrapper.transaction_index, - log_index: transfer_wrapper.log_index, - value: transfer_wrapper.value.map(|v| v.0), contract_type, - transaction_type: transfer_wrapper.transaction_type, - token_address: transfer_wrapper.token_address, - token_id: transfer_wrapper.token_id.0, token_uri: None, collection_name: None, image_url: None, token_name: None, - from_address: transfer_wrapper.from_address, - to_address: transfer_wrapper.to_address, status, - amount: transfer_wrapper.amount.0, - verified: transfer_wrapper.verified, - operator: transfer_wrapper.operator, - possible_spam: transfer_wrapper.possible_spam, }; // collect NFTs transfers from the page res_list.push(transfer_history); @@ -368,31 +372,33 @@ async fn get_moralis_metadata( drop_mutability!(uri); let response = send_request_to_uri(uri.as_str()).await?; - let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string())?; - let contract_type = match nft_wrapper.contract_type { - Some(contract_type) => contract_type.0, + let nft_moralis: NftFromMoralis = serde_json::from_str(&response.to_string())?; + let contract_type = match nft_moralis.contract_type { + Some(contract_type) => contract_type, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let token_uri = check_moralis_ipfs_bafy(nft_wrapper.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), nft_wrapper.metadata.as_deref()).await; + let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; let nft_metadata = Nft { + common: NftCommon { + token_address: nft_moralis.common.token_address, + token_id: nft_moralis.common.token_id, + amount: nft_moralis.common.amount, + owner_of: nft_moralis.common.owner_of, + token_hash: nft_moralis.common.token_hash, + collection_name: nft_moralis.common.collection_name, + symbol: nft_moralis.common.symbol, + token_uri, + metadata: nft_moralis.common.metadata, + last_token_uri_sync: nft_moralis.common.last_token_uri_sync, + last_metadata_sync: nft_moralis.common.last_metadata_sync, + minter_address: nft_moralis.common.minter_address, + possible_spam: nft_moralis.common.possible_spam, + }, chain: *chain, - token_address: nft_wrapper.token_address, - token_id: nft_wrapper.token_id.0, - amount: nft_wrapper.amount.0, - owner_of: nft_wrapper.owner_of, - token_hash: nft_wrapper.token_hash, - block_number_minted: nft_wrapper.block_number_minted.map(|v| v.0), - block_number: *nft_wrapper.block_number, + block_number_minted: nft_moralis.block_number_minted.map(|v| v.0), + block_number: *nft_moralis.block_number, contract_type, - collection_name: nft_wrapper.name, - symbol: nft_wrapper.symbol, - token_uri, - metadata: nft_wrapper.metadata, - last_token_uri_sync: nft_wrapper.last_token_uri_sync, - last_metadata_sync: nft_wrapper.last_metadata_sync, - minter_address: nft_wrapper.minter_address, - possible_spam: nft_wrapper.possible_spam, uri_meta, }; Ok(nft_metadata) @@ -555,16 +561,16 @@ async fn handle_send_erc721( tx: NftTransferHistory, ) -> MmResult<(), UpdateNftError> { let nft_db = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .get_nft(chain, tx.common.token_address.clone(), tx.common.token_id.clone()) .await? .ok_or_else(|| UpdateNftError::TokenNotFoundInWallet { - token_address: tx.token_address.clone(), - token_id: tx.token_id.to_string(), + token_address: tx.common.token_address.clone(), + token_id: tx.common.token_id.to_string(), })?; let tx_meta = TxMeta::from(nft_db); storage.update_txs_meta_by_token_addr_id(chain, tx_meta).await?; storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .remove_nft_from_list(chain, tx.common.token_address, tx.common.token_id, tx.block_number) .await?; Ok(()) } @@ -577,15 +583,15 @@ async fn handle_receive_erc721( my_address: &str, ) -> MmResult<(), UpdateNftError> { let nft = match storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .get_nft(chain, tx.common.token_address.clone(), tx.common.token_id.clone()) .await? { Some(mut nft_db) => { // An error is raised if user tries to receive an identical ERC-721 token they already own // and if owner address != from address - if my_address != tx.from_address { + if my_address != tx.common.from_address { return MmError::err(UpdateNftError::AttemptToReceiveAlreadyOwnedErc721 { - tx_hash: tx.transaction_hash, + tx_hash: tx.common.transaction_hash, }); } nft_db.block_number = tx.block_number; @@ -597,10 +603,10 @@ async fn handle_receive_erc721( }, // If token isn't in NFT LIST table then add nft to the table. None => { - let mut nft = get_moralis_metadata(tx.token_address, tx.token_id, chain, url).await?; + let mut nft = get_moralis_metadata(tx.common.token_address, tx.common.token_id, chain, url).await?; // sometimes moralis updates Get All NFTs (which also affects Get Metadata) later // than History by Wallet update - nft.owner_of = my_address.to_string(); + nft.common.owner_of = my_address.to_string(); nft.block_number = tx.block_number; drop_mutability!(nft); storage @@ -620,28 +626,28 @@ async fn handle_send_erc1155( tx: NftTransferHistory, ) -> MmResult<(), UpdateNftError> { let mut nft_db = storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .get_nft(chain, tx.common.token_address.clone(), tx.common.token_id.clone()) .await? .ok_or_else(|| UpdateNftError::TokenNotFoundInWallet { - token_address: tx.token_address.clone(), - token_id: tx.token_id.to_string(), + token_address: tx.common.token_address.clone(), + token_id: tx.common.token_id.to_string(), })?; - match nft_db.amount.cmp(&tx.amount) { + match nft_db.common.amount.cmp(&tx.common.amount) { Ordering::Equal => { storage - .remove_nft_from_list(chain, tx.token_address, tx.token_id, tx.block_number) + .remove_nft_from_list(chain, tx.common.token_address, tx.common.token_id, tx.block_number) .await?; }, Ordering::Greater => { - nft_db.amount -= tx.amount; + nft_db.common.amount -= tx.common.amount; storage .update_nft_amount(chain, nft_db.clone(), tx.block_number) .await?; }, Ordering::Less => { return MmError::err(UpdateNftError::InsufficientAmountInCache { - amount_list: nft_db.amount.to_string(), - amount_history: tx.amount.to_string(), + amount_list: nft_db.common.amount.to_string(), + amount_history: tx.common.amount.to_string(), }); }, } @@ -658,14 +664,14 @@ async fn handle_receive_erc1155( my_address: &str, ) -> MmResult<(), UpdateNftError> { let nft = match storage - .get_nft(chain, tx.token_address.clone(), tx.token_id.clone()) + .get_nft(chain, tx.common.token_address.clone(), tx.common.token_id.clone()) .await? { Some(mut nft_db) => { // if owner address == from address, then owner sent tokens to themself, // which means that the amount will not change. - if my_address != tx.from_address { - nft_db.amount += tx.amount; + if my_address != tx.common.from_address { + nft_db.common.amount += tx.common.amount; } nft_db.block_number = tx.block_number; drop_mutability!(nft_db); @@ -676,27 +682,30 @@ async fn handle_receive_erc1155( }, // If token isn't in NFT LIST table then add nft to the table. None => { - let moralis_meta = get_moralis_metadata(tx.token_address, tx.token_id.clone(), chain, url).await?; - let token_uri = check_moralis_ipfs_bafy(moralis_meta.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.metadata.as_deref()).await; + let moralis_meta = + get_moralis_metadata(tx.common.token_address, tx.common.token_id.clone(), chain, url).await?; + let token_uri = check_moralis_ipfs_bafy(moralis_meta.common.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), moralis_meta.common.metadata.as_deref()).await; let nft = Nft { + common: NftCommon { + token_address: moralis_meta.common.token_address, + token_id: moralis_meta.common.token_id, + amount: tx.common.amount, + owner_of: my_address.to_string(), + token_hash: moralis_meta.common.token_hash, + collection_name: moralis_meta.common.collection_name, + symbol: moralis_meta.common.symbol, + token_uri, + metadata: moralis_meta.common.metadata, + last_token_uri_sync: moralis_meta.common.last_token_uri_sync, + last_metadata_sync: moralis_meta.common.last_metadata_sync, + minter_address: moralis_meta.common.minter_address, + possible_spam: moralis_meta.common.possible_spam, + }, chain: *chain, - token_address: moralis_meta.token_address, - token_id: moralis_meta.token_id, - amount: tx.amount, - owner_of: my_address.to_string(), - token_hash: moralis_meta.token_hash, block_number_minted: moralis_meta.block_number_minted, block_number: tx.block_number, contract_type: moralis_meta.contract_type, - collection_name: moralis_meta.collection_name, - symbol: moralis_meta.symbol, - token_uri, - metadata: moralis_meta.metadata, - last_token_uri_sync: moralis_meta.last_token_uri_sync, - last_metadata_sync: moralis_meta.last_metadata_sync, - minter_address: moralis_meta.minter_address, - possible_spam: moralis_meta.possible_spam, uri_meta, }; storage.add_nfts_to_list(chain, [nft.clone()], tx.block_number).await?; @@ -727,7 +736,7 @@ pub(crate) async fn find_wallet_nft_amount( token_address, token_id: token_id.to_string(), })?; - Ok(nft_meta.amount) + Ok(nft_meta.common.amount) } async fn cache_nfts_from_moralis( @@ -799,7 +808,7 @@ fn protect_from_history_spam(tx: &mut NftTransferHistory) { let token_name_spam = check_and_redact_if_spam(&mut tx.token_name); if collection_name_spam || token_name_spam { - tx.possible_spam = true; + tx.common.possible_spam = true; } } /// `protect_from_nft_spam` function checks and redact spam in `Nft`. @@ -808,13 +817,13 @@ fn protect_from_history_spam(tx: &mut NftTransferHistory) { /// they must be just an arbitrary text, which represents NFT names. /// `symbol` also must be a text or sign that represents a symbol. fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { - let collection_name_spam = check_and_redact_if_spam(&mut nft.collection_name); - let symbol_spam = check_and_redact_if_spam(&mut nft.symbol); + let collection_name_spam = check_and_redact_if_spam(&mut nft.common.collection_name); + let symbol_spam = check_and_redact_if_spam(&mut nft.common.symbol); let token_name_spam = check_and_redact_if_spam(&mut nft.uri_meta.token_name); let meta_spam = check_nft_metadata_for_spam(nft)?; if collection_name_spam || symbol_spam || token_name_spam || meta_spam { - nft.possible_spam = true; + nft.common.possible_spam = true; } Ok(()) } @@ -823,12 +832,13 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { /// **note:** `token_name` is usually called `name` in `metadata`. fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { if let Some(Ok(mut metadata)) = nft + .common .metadata .as_ref() .map(|t| serde_json::from_str::>(t)) { if check_spam_and_redact_metadata_field(&mut metadata, "name") { - nft.metadata = Some(serde_json::to_string(&metadata)?); + nft.common.metadata = Some(serde_json::to_string(&metadata)?); return Ok(true); } } diff --git a/mm2src/coins/nft/nft_structs.rs b/mm2src/coins/nft/nft_structs.rs index 3f50749c5b..444fec469f 100644 --- a/mm2src/coins/nft/nft_structs.rs +++ b/mm2src/coins/nft/nft_structs.rs @@ -185,17 +185,15 @@ impl UriMeta { } } +/// [`NftCommon`] structure contains common fields from [`Nft`] and [`NftFromMoralis`] #[derive(Clone, Debug, Deserialize, Serialize)] -pub struct Nft { - pub(crate) chain: Chain, +pub struct NftCommon { pub(crate) token_address: String, pub(crate) token_id: BigDecimal, pub(crate) amount: BigDecimal, pub(crate) owner_of: String, pub(crate) token_hash: Option, - pub(crate) block_number_minted: Option, - pub(crate) block_number: u64, - pub(crate) contract_type: ContractType, + #[serde(rename = "name")] pub(crate) collection_name: Option, pub(crate) symbol: Option, pub(crate) token_uri: Option, @@ -203,31 +201,29 @@ pub struct Nft { pub(crate) last_token_uri_sync: Option, pub(crate) last_metadata_sync: Option, pub(crate) minter_address: Option, + #[serde(default)] pub(crate) possible_spam: bool, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct Nft { + #[serde(flatten)] + pub(crate) common: NftCommon, + pub(crate) chain: Chain, + pub(crate) block_number_minted: Option, + pub(crate) block_number: u64, + pub(crate) contract_type: ContractType, pub(crate) uri_meta: UriMeta, } -/// This structure is for deserializing NFT json to struct. -/// Its needed to convert fields properly, because all fields in json have string type. +/// This structure is for deserializing moralis NFT json to struct. #[derive(Debug, Deserialize)] -pub(crate) struct NftWrapper { - pub(crate) token_address: String, - pub(crate) token_id: SerdeStringWrap, - pub(crate) amount: SerdeStringWrap, - pub(crate) owner_of: String, - pub(crate) token_hash: Option, +pub(crate) struct NftFromMoralis { + #[serde(flatten)] + pub(crate) common: NftCommon, pub(crate) block_number_minted: Option>, pub(crate) block_number: SerdeStringWrap, - pub(crate) contract_type: Option>, - pub(crate) name: Option, - pub(crate) symbol: Option, - pub(crate) token_uri: Option, - pub(crate) metadata: Option, - pub(crate) last_token_uri_sync: Option, - pub(crate) last_metadata_sync: Option, - pub(crate) minter_address: Option, - #[serde(default)] - pub(crate) possible_spam: bool, + pub(crate) contract_type: Option, } #[derive(Debug)] @@ -362,55 +358,50 @@ impl fmt::Display for TransferStatus { } } +/// [`NftTransferCommon`] structure contains common fields from [`NftTransferHistory`] and [`NftTxHistoryFromMoralis`] #[derive(Clone, Debug, Deserialize, Serialize)] -pub struct NftTransferHistory { - pub(crate) chain: Chain, - pub(crate) block_number: u64, - pub(crate) block_timestamp: u64, +pub struct NftTransferCommon { pub(crate) block_hash: Option, /// Transaction hash in hexadecimal format pub(crate) transaction_hash: String, pub(crate) transaction_index: Option, pub(crate) log_index: Option, pub(crate) value: Option, - pub(crate) contract_type: ContractType, pub(crate) transaction_type: Option, pub(crate) token_address: String, pub(crate) token_id: BigDecimal, - pub(crate) token_uri: Option, - pub(crate) collection_name: Option, - pub(crate) image_url: Option, - pub(crate) token_name: Option, pub(crate) from_address: String, pub(crate) to_address: String, - pub(crate) status: TransferStatus, pub(crate) amount: BigDecimal, pub(crate) verified: Option, pub(crate) operator: Option, + #[serde(default)] pub(crate) possible_spam: bool, } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct NftTransferHistory { + #[serde(flatten)] + pub(crate) common: NftTransferCommon, + pub(crate) chain: Chain, + pub(crate) block_number: u64, + pub(crate) block_timestamp: u64, + pub(crate) contract_type: ContractType, + pub(crate) token_uri: Option, + pub(crate) collection_name: Option, + pub(crate) image_url: Option, + pub(crate) token_name: Option, + pub(crate) status: TransferStatus, +} + +/// This structure is for deserializing moralis NFT transaction json to struct. #[derive(Debug, Deserialize)] -pub(crate) struct NftTransferHistoryWrapper { +pub(crate) struct NftTxHistoryFromMoralis { + #[serde(flatten)] + pub(crate) common: NftTransferCommon, pub(crate) block_number: SerdeStringWrap, pub(crate) block_timestamp: String, - pub(crate) block_hash: Option, - /// Transaction hash in hexadecimal format - pub(crate) transaction_hash: String, - pub(crate) transaction_index: Option, - pub(crate) log_index: Option, - pub(crate) value: Option>, - pub(crate) contract_type: Option>, - pub(crate) transaction_type: Option, - pub(crate) token_address: String, - pub(crate) token_id: SerdeStringWrap, - pub(crate) from_address: String, - pub(crate) to_address: String, - pub(crate) amount: SerdeStringWrap, - pub(crate) verified: Option, - pub(crate) operator: Option, - #[serde(default)] - pub(crate) possible_spam: bool, + pub(crate) contract_type: Option, } #[derive(Debug, Serialize)] @@ -455,10 +446,10 @@ pub struct TxMeta { impl From for TxMeta { fn from(nft_db: Nft) -> Self { TxMeta { - token_address: nft_db.token_address, - token_id: nft_db.token_id, - token_uri: nft_db.token_uri, - collection_name: nft_db.collection_name, + token_address: nft_db.common.token_address, + token_id: nft_db.common.token_id, + token_uri: nft_db.common.token_uri, + collection_name: nft_db.common.collection_name, image_url: nft_db.uri_meta.image_url, token_name: nft_db.uri_meta.token_name, } diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 97704adee3..94c3abe3e2 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -5,7 +5,7 @@ const TEST_WALLET_ADDR_EVM: &str = "0x394d86994f954ed931b86791b62fe64f4c5dac37"; #[cfg(all(test, not(target_arch = "wasm32")))] mod native_tests { - use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper, UriMeta}; + use crate::nft::nft_structs::{NftFromMoralis, NftTxHistoryFromMoralis, UriMeta}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::storage::db_test_helpers::*; use crate::nft::{check_and_redact_if_spam, check_moralis_ipfs_bafy, check_nft_metadata_for_spam, @@ -38,35 +38,29 @@ mod native_tests { let mut nft = nft(); assert!(check_nft_metadata_for_spam(&mut nft).unwrap()); let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; - assert_eq!(meta_redacted, nft.metadata.unwrap()) + assert_eq!(meta_redacted, nft.common.metadata.unwrap()) } #[test] - fn test_moralis_nft_list() { - let response = block_on(send_request_to_uri(NFT_LIST_URL_TEST)).unwrap(); - let nfts_list = response["result"].as_array().unwrap(); + fn test_moralis_requests() { + let response_nft_list = block_on(send_request_to_uri(NFT_LIST_URL_TEST)).unwrap(); + let nfts_list = response_nft_list["result"].as_array().unwrap(); for nft_json in nfts_list { - let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, nft_wrapper.owner_of); + let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); + assert_eq!(TEST_WALLET_ADDR_EVM, nft_moralis.common.owner_of); } - } - #[test] - fn test_moralis_nft_transfer_history() { - let response = block_on(send_request_to_uri(NFT_HISTORY_URL_TEST)).unwrap(); - let mut transfer_list = response["result"].as_array().unwrap().clone(); + let response_tx_history = block_on(send_request_to_uri(NFT_HISTORY_URL_TEST)).unwrap(); + let mut transfer_list = response_tx_history["result"].as_array().unwrap().clone(); assert!(!transfer_list.is_empty()); let first_tx = transfer_list.remove(transfer_list.len() - 1); - let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&first_tx.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, transfer_wrapper.to_address); - } + let transfer_moralis: NftTxHistoryFromMoralis = serde_json::from_str(&first_tx.to_string()).unwrap(); + assert_eq!(TEST_WALLET_ADDR_EVM, transfer_moralis.common.to_address); - #[test] - fn test_moralis_nft_metadata() { - let response = block_on(send_request_to_uri(NFT_METADATA_URL_TEST)).unwrap(); - let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); - assert_eq!(41237364, *nft_wrapper.block_number_minted.unwrap()); - let token_uri = nft_wrapper.token_uri.unwrap(); + let response_meta = block_on(send_request_to_uri(NFT_METADATA_URL_TEST)).unwrap(); + let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); + assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); + let token_uri = nft_moralis.common.token_uri.unwrap(); let uri_response = block_on(send_request_to_uri(token_uri.as_str())).unwrap(); serde_json::from_str::(&uri_response.to_string()).unwrap(); } @@ -107,7 +101,7 @@ mod native_tests { #[cfg(target_arch = "wasm32")] mod wasm_tests { - use crate::nft::nft_structs::{NftTransferHistoryWrapper, NftWrapper}; + use crate::nft::nft_structs::{NftFromMoralis, NftTxHistoryFromMoralis}; use crate::nft::nft_tests::{NFT_HISTORY_URL_TEST, NFT_LIST_URL_TEST, NFT_METADATA_URL_TEST, TEST_WALLET_ADDR_EVM}; use crate::nft::send_request_to_uri; use crate::nft::storage::db_test_helpers::*; @@ -116,30 +110,24 @@ mod wasm_tests { wasm_bindgen_test_configure!(run_in_browser); #[wasm_bindgen_test] - async fn test_moralis_nft_list() { - let response = send_request_to_uri(NFT_LIST_URL_TEST).await.unwrap(); - let nfts_list = response["result"].as_array().unwrap(); + async fn test_moralis_requests() { + let response_nft_list = send_request_to_uri(NFT_LIST_URL_TEST).await.unwrap(); + let nfts_list = response_nft_list["result"].as_array().unwrap(); for nft_json in nfts_list { - let nft_wrapper: NftWrapper = serde_json::from_str(&nft_json.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, nft_wrapper.owner_of); + let nft_moralis: NftFromMoralis = serde_json::from_str(&nft_json.to_string()).unwrap(); + assert_eq!(TEST_WALLET_ADDR_EVM, nft_moralis.common.owner_of); } - } - #[wasm_bindgen_test] - async fn test_moralis_nft_transfer_history() { - let response = send_request_to_uri(NFT_HISTORY_URL_TEST).await.unwrap(); - let mut transfer_list = response["result"].as_array().unwrap().clone(); + let response_tx_history = send_request_to_uri(NFT_HISTORY_URL_TEST).await.unwrap(); + let mut transfer_list = response_tx_history["result"].as_array().unwrap().clone(); assert!(!transfer_list.is_empty()); let first_tx = transfer_list.remove(transfer_list.len() - 1); - let transfer_wrapper: NftTransferHistoryWrapper = serde_json::from_str(&first_tx.to_string()).unwrap(); - assert_eq!(TEST_WALLET_ADDR_EVM, transfer_wrapper.to_address); - } + let transfer_moralis: NftTxHistoryFromMoralis = serde_json::from_str(&first_tx.to_string()).unwrap(); + assert_eq!(TEST_WALLET_ADDR_EVM, transfer_moralis.common.to_address); - #[wasm_bindgen_test] - async fn test_moralis_nft_metadata() { - let response = send_request_to_uri(NFT_METADATA_URL_TEST).await.unwrap(); - let nft_wrapper: NftWrapper = serde_json::from_str(&response.to_string()).unwrap(); - assert_eq!(41237364, *nft_wrapper.block_number_minted.unwrap()); + let response_meta = send_request_to_uri(NFT_METADATA_URL_TEST).await.unwrap(); + let nft_moralis: NftFromMoralis = serde_json::from_str(&response_meta.to_string()).unwrap(); + assert_eq!(41237364, *nft_moralis.block_number_minted.unwrap()); } #[wasm_bindgen_test] diff --git a/mm2src/coins/nft/storage/db_test_helpers.rs b/mm2src/coins/nft/storage/db_test_helpers.rs index fcb00d4893..e744538e15 100644 --- a/mm2src/coins/nft/storage/db_test_helpers.rs +++ b/mm2src/coins/nft/storage/db_test_helpers.rs @@ -1,5 +1,5 @@ -use crate::nft::nft_structs::{Chain, ContractType, Nft, NftTransferHistory, NftTxHistoryFilters, TransferStatus, - TxMeta, UriMeta}; +use crate::nft::nft_structs::{Chain, ContractType, Nft, NftCommon, NftTransferCommon, NftTransferHistory, + NftTxHistoryFilters, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps, RemoveNftResult}; use mm2_number::BigDecimal; use mm2_test_helpers::for_tests::mm_ctx_with_custom_db; @@ -18,26 +18,29 @@ const TX_HASH: &str = "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9 pub(crate) fn nft() -> Nft { Nft { + common: NftCommon { + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + amount: BigDecimal::from_str("2").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), + collection_name: None, + symbol: None, + token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + metadata: Some( + "{\"name\":\"https://arweave.net\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), + last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), + minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), + possible_spam: false, + }, chain: Chain::Bsc, - token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), - token_id: Default::default(), - amount: BigDecimal::from_str("2").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), block_number_minted: Some(25465916), block_number: 25919780, contract_type: ContractType::Erc1155, - collection_name: None, - symbol: None, - token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), - metadata: Some( - "{\"name\":\"https://arweave.net\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}" - .to_string(), - ), - last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), - last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), - minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: false, + uri_meta: UriMeta { image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), raw_image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), @@ -53,51 +56,55 @@ pub(crate) fn nft() -> Nft { fn tx() -> NftTransferHistory { NftTransferHistory { + common: NftTransferCommon { + block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), + transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), + transaction_index: Some(198), + log_index: Some(495), + value: Default::default(), + transaction_type: Some("Single".to_string()), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + amount: BigDecimal::from_str("1").unwrap(), + verified: Some(1), + operator: None, + possible_spam: false, + }, chain: Chain::Bsc, block_number: 28056726, block_timestamp: 1683627432, - block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), - transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), - transaction_index: Some(198), - log_index: Some(495), - value: Default::default(), contract_type: ContractType::Erc721, - transaction_type: Some("Single".to_string()), - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300047252").unwrap(), token_uri: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), token_name: Some("Nebula Nodes".to_string()), - from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: Some(1), - operator: None, - possible_spam: false, } } fn nft_list() -> Vec { let nft = Nft { + common: NftCommon { + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + amount: BigDecimal::from_str("2").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), + collection_name: None, + symbol: None, + token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), + metadata: Some("{\"name\":\"Tiki box\"}".to_string()), + last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), + last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), + minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), + possible_spam: false, + }, chain: Chain::Bsc, - token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), - token_id: Default::default(), - amount: BigDecimal::from_str("2").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: Some("b34ddf294013d20a6d70691027625839".to_string()), block_number_minted: Some(25465916), block_number: 25919780, contract_type: ContractType::Erc1155, - collection_name: None, - symbol: None, - token_uri: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.json".to_string()), - metadata: Some("{\"name\":\"Tiki box\"}".to_string()), - last_token_uri_sync: Some("2023-02-07T17:10:08.402Z".to_string()), - last_metadata_sync: Some("2023-02-07T17:10:16.858Z".to_string()), - minter_address: Some("ERC1155 tokens don't have a single minter".to_string()), - possible_spam: false, uri_meta: UriMeta { image_url: Some("https://tikimetadata.s3.amazonaws.com/tiki_box.png".to_string()), raw_image_url: None, @@ -111,26 +118,29 @@ fn nft_list() -> Vec { }; let nft1 = Nft { + common: NftCommon { + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: Some("c5d1cfd75a0535b0ec750c0156e6ddfe".to_string()), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + symbol: Some("BMBBBF".to_string()), + token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), + metadata: Some( + "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), + last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), + minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), + possible_spam: false, + }, chain: Chain::Bsc, - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300047252").unwrap(), - amount: BigDecimal::from_str("1").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: Some("c5d1cfd75a0535b0ec750c0156e6ddfe".to_string()), + block_number_minted: Some(25721963), block_number: 28056726, contract_type: ContractType::Erc721, - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - symbol: Some("BMBBBF".to_string()), - token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300047252".to_string()), - metadata: Some( - "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" - .to_string(), - ), - last_token_uri_sync: Some("2023-02-16T16:35:52.392Z".to_string()), - last_metadata_sync: Some("2023-02-16T16:36:04.283Z".to_string()), - minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: false, uri_meta: UriMeta { image_url: Some( "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), @@ -146,26 +156,29 @@ fn nft_list() -> Vec { }; let nft2 = Nft { + common: NftCommon { + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300044414").unwrap(), + amount: BigDecimal::from_str("1").unwrap(), + owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + token_hash: Some("125f8f4e952e107c257960000b4b250e".to_string()), + collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), + symbol: Some("BMBBBF".to_string()), + token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300044414".to_string()), + metadata: Some( + "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" + .to_string(), + ), + last_token_uri_sync: Some("2023-02-19T19:12:09.471Z".to_string()), + last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), + minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), + possible_spam: false, + }, chain: Chain::Bsc, - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300044414").unwrap(), - amount: BigDecimal::from_str("1").unwrap(), - owner_of: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), - token_hash: Some("125f8f4e952e107c257960000b4b250e".to_string()), + block_number_minted: Some(25810308), block_number: 28056721, contract_type: ContractType::Erc721, - collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), - symbol: Some("BMBBBF".to_string()), - token_uri: Some("https://public.nftstatic.com/static/nft/BSC/BMBBBF/214300044414".to_string()), - metadata: Some( - "{\"image\":\"https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png\"}" - .to_string(), - ), - last_token_uri_sync: Some("2023-02-19T19:12:09.471Z".to_string()), - last_metadata_sync: Some("2023-02-19T19:12:18.080Z".to_string()), - minter_address: Some("0xdbdeb0895f3681b87fb3654b5cf3e05546ba24a9".to_string()), - possible_spam: false, uri_meta: UriMeta { image_url: Some( "https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string(), @@ -184,81 +197,92 @@ fn nft_list() -> Vec { fn nft_tx_historty() -> Vec { let tx = NftTransferHistory { + common: NftTransferCommon { + block_hash: Some("0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string()), + transaction_hash: "0x9c16b962f63eead1c5d2355cc9037dde178b14b53043c57eb40c27964d22ae6a".to_string(), + transaction_index: Some(57), + log_index: Some(139), + value: Default::default(), + transaction_type: Some("Single".to_string()), + token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), + token_id: Default::default(), + from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + amount: BigDecimal::from_str("1").unwrap(), + verified: Some(1), + operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), + possible_spam: false, + }, chain: Chain::Bsc, block_number: 25919780, block_timestamp: 1677166110, - block_hash: Some("0xcb41654fc5cf2bf5d7fd3f061693405c74d419def80993caded0551ecfaeaae5".to_string()), - transaction_hash: "0x9c16b962f63eead1c5d2355cc9037dde178b14b53043c57eb40c27964d22ae6a".to_string(), - transaction_index: Some(57), - log_index: Some(139), - value: Default::default(), contract_type: ContractType::Erc1155, - transaction_type: Some("Single".to_string()), - token_address: "0x5c7d6712dfaf0cb079d48981781c8705e8417ca0".to_string(), - token_id: Default::default(), token_uri: None, collection_name: None, image_url: None, token_name: None, - from_address: "0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: Some(1), - operator: Some("0x4ff0bbc9b64d635a4696d1a38554fb2529c103ff".to_string()), - possible_spam: false, }; let tx1 = NftTransferHistory { + common: NftTransferCommon { + block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), + transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), + transaction_index: Some(198), + log_index: Some(495), + value: Default::default(), + transaction_type: Some("Single".to_string()), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300047252").unwrap(), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + amount: BigDecimal::from_str("1").unwrap(), + verified: Some(1), + operator: None, + possible_spam: false, + }, chain: Chain::Bsc, block_number: 28056726, block_timestamp: 1683627432, - block_hash: Some("0x3d68b78391fb3cf8570df27036214f7e9a5a6a45d309197936f51d826041bfe7".to_string()), - transaction_hash: "0x1e9f04e9b571b283bde02c98c2a97da39b2bb665b57c1f2b0b733f9b681debbe".to_string(), - transaction_index: Some(198), - log_index: Some(495), - value: Default::default(), contract_type: ContractType::Erc721, - transaction_type: Some("Single".to_string()), - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300047252").unwrap(), + token_uri: None, collection_name: None, image_url: None, token_name: None, - from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: Some(1), - operator: None, - possible_spam: false, }; let tx2 = NftTransferHistory { + common: NftTransferCommon { + block_hash: Some("0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string()), + transaction_hash: "0x981bad702cc6e088f0e9b5e7287ff7a3487b8d269103cee3b9e5803141f63f91".to_string(), + transaction_index: Some(83), + log_index: Some(201), + value: Default::default(), + transaction_type: Some("Single".to_string()), + token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), + token_id: BigDecimal::from_str("214300044414").unwrap(), + from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), + to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + amount: BigDecimal::from_str("1").unwrap(), + verified: Some(1), + operator: None, + possible_spam: false, + }, chain: Chain::Bsc, block_number: 28056721, block_timestamp: 1683627417, - block_hash: Some("0x326db41c5a4fd5f033676d95c590ced18936ef2ef6079e873b23af087fd966c6".to_string()), - transaction_hash: "0x981bad702cc6e088f0e9b5e7287ff7a3487b8d269103cee3b9e5803141f63f91".to_string(), - transaction_index: Some(83), - log_index: Some(201), - value: Default::default(), + contract_type: ContractType::Erc721, - transaction_type: Some("Single".to_string()), - token_address: "0xfd913a305d70a60aac4faac70c739563738e1f81".to_string(), - token_id: BigDecimal::from_str("214300044414").unwrap(), + token_uri: None, collection_name: Some("Binance NFT Mystery Box-Back to Blockchain Future".to_string()), image_url: Some("https://public.nftstatic.com/static/nft/res/4df0a5da04174e1e9be04b22a805f605.png".to_string()), token_name: Some("Nebula Nodes".to_string()), - from_address: "0x6fad0ec6bb76914b2a2a800686acc22970645820".to_string(), - to_address: "0xf622a6c52c94b500542e2ae6bcad24c53bc5b6a2".to_string(), + status: TransferStatus::Receive, - amount: BigDecimal::from_str("1").unwrap(), - verified: Some(1), - operator: None, - possible_spam: false, }; vec![tx, tx1, tx2] } @@ -360,10 +384,10 @@ pub(crate) async fn test_nft_amount_impl() { .await .unwrap(); - nft.amount -= BigDecimal::from(1); + nft.common.amount -= BigDecimal::from(1); storage.update_nft_amount(&chain, nft.clone(), 25919800).await.unwrap(); let amount = storage - .get_nft_amount(&chain, nft.token_address.clone(), nft.token_id.clone()) + .get_nft_amount(&chain, nft.common.token_address.clone(), nft.common.token_id.clone()) .await .unwrap() .unwrap(); @@ -371,14 +395,14 @@ pub(crate) async fn test_nft_amount_impl() { let last_scanned_block = storage.get_last_scanned_block(&chain).await.unwrap().unwrap(); assert_eq!(last_scanned_block, 25919800); - nft.amount += BigDecimal::from(1); + nft.common.amount += BigDecimal::from(1); nft.block_number = 25919900; storage .update_nft_amount_and_block_number(&chain, nft.clone()) .await .unwrap(); let amount = storage - .get_nft_amount(&chain, nft.token_address, nft.token_id) + .get_nft_amount(&chain, nft.common.token_address, nft.common.token_id) .await .unwrap() .unwrap(); @@ -396,13 +420,13 @@ pub(crate) async fn test_refresh_metadata_impl() { .add_nfts_to_list(&chain, vec![nft.clone()], 25919780) .await .unwrap(); - nft.symbol = Some(new_symbol.to_string()); + nft.common.symbol = Some(new_symbol.to_string()); drop_mutability!(nft); - let token_add = nft.token_address.clone(); - let token_id = nft.token_id.clone(); + let token_add = nft.common.token_address.clone(); + let token_id = nft.common.token_id.clone(); storage.refresh_nft_metadata(&chain, nft).await.unwrap(); let nft_upd = storage.get_nft(&chain, token_add, token_id).await.unwrap().unwrap(); - assert_eq!(new_symbol.to_string(), nft_upd.symbol.unwrap()); + assert_eq!(new_symbol.to_string(), nft_upd.common.symbol.unwrap()); } pub(crate) async fn test_add_get_txs_impl() { diff --git a/mm2src/coins/nft/storage/sql_storage.rs b/mm2src/coins/nft/storage/sql_storage.rs index 6705a2ab8f..2e319ba5fa 100644 --- a/mm2src/coins/nft/storage/sql_storage.rs +++ b/mm2src/coins/nft/storage/sql_storage.rs @@ -510,10 +510,10 @@ impl NftListStorageOps for SqliteNftStorage { for nft in nfts { let nft_json = json::to_string(&nft).expect("serialization should not fail"); let params = [ - Some(nft.token_address), - Some(nft.token_id.to_string()), + Some(nft.common.token_address), + Some(nft.common.token_id.to_string()), Some(nft.chain.to_string()), - Some(nft.amount.to_string()), + Some(nft.common.amount.to_string()), Some(nft.block_number.to_string()), Some(nft.contract_type.to_string()), Some(nft_json), @@ -595,7 +595,7 @@ impl NftListStorageOps for SqliteNftStorage { async_blocking(move || { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; - let params = [nft_json, nft.token_address, nft.token_id.to_string()]; + let params = [nft_json, nft.common.token_address, nft.common.token_id.to_string()]; sql_transaction.execute(&sql, params)?; sql_transaction.commit()?; Ok(()) @@ -639,10 +639,10 @@ impl NftListStorageOps for SqliteNftStorage { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; let params = [ - Some(nft.amount.to_string()), + Some(nft.common.amount.to_string()), Some(nft_json), - Some(nft.token_address), - Some(nft.token_id.to_string()), + Some(nft.common.token_address), + Some(nft.common.token_id.to_string()), ]; sql_transaction.execute(&sql, params)?; sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; @@ -661,11 +661,11 @@ impl NftListStorageOps for SqliteNftStorage { let mut conn = selfi.0.lock().unwrap(); let sql_transaction = conn.transaction()?; let params = [ - Some(nft.amount.to_string()), + Some(nft.common.amount.to_string()), Some(nft.block_number.to_string()), Some(nft_json), - Some(nft.token_address), - Some(nft.token_id.to_string()), + Some(nft.common.token_address), + Some(nft.common.token_id.to_string()), ]; sql_transaction.execute(&sql, params)?; sql_transaction.execute(&upsert_last_scanned_block_sql()?, scanned_block_params)?; @@ -755,15 +755,15 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { for tx in txs { let tx_json = json::to_string(&tx).expect("serialization should not fail"); let params = [ - Some(tx.transaction_hash), + Some(tx.common.transaction_hash), Some(tx.chain.to_string()), Some(tx.block_number.to_string()), Some(tx.block_timestamp.to_string()), Some(tx.contract_type.to_string()), - Some(tx.token_address), - Some(tx.token_id.to_string()), + Some(tx.common.token_address), + Some(tx.common.token_id.to_string()), Some(tx.status.to_string()), - Some(tx.amount.to_string()), + Some(tx.common.amount.to_string()), tx.collection_name, tx.image_url, tx.token_name, @@ -846,7 +846,7 @@ impl NftTxHistoryStorageOps for SqliteNftStorage { tx.image_url, tx.token_name, Some(tx_json), - Some(tx.transaction_hash), + Some(tx.common.transaction_hash), ]; let selfi = self.clone(); async_blocking(move || { diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index eb5a2404a4..89237f1ba2 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -224,7 +224,7 @@ impl NftListStorageOps for IndexedDbNftStorage { .with_value(token_id.to_string())?; if let Some((_item_id, item)) = table.get_item_by_unique_multi_index(index_keys).await? { - Ok(Some(nft_details_from_item(item)?.amount.to_string())) + Ok(Some(nft_details_from_item(item)?.common.amount.to_string())) } else { Ok(None) } @@ -236,8 +236,8 @@ impl NftListStorageOps for IndexedDbNftStorage { let table = db_transaction.table::().await?; let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? - .with_value(&nft.token_address)? - .with_value(nft.token_id.to_string())?; + .with_value(&nft.common.token_address)? + .with_value(nft.common.token_id.to_string())?; let nft_item = NftListTable::from_nft(&nft)?; table.replace_item_by_unique_multi_index(index_keys, &nft_item).await?; @@ -274,8 +274,8 @@ impl NftListStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? - .with_value(&nft.token_address)? - .with_value(nft.token_id.to_string())?; + .with_value(&nft.common.token_address)? + .with_value(nft.common.token_id.to_string())?; let nft_item = NftListTable::from_nft(&nft)?; nft_table @@ -299,8 +299,8 @@ impl NftListStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftListTable::CHAIN_TOKEN_ADD_TOKEN_ID_INDEX) .with_value(chain.to_string())? - .with_value(&nft.token_address)? - .with_value(nft.token_id.to_string())?; + .with_value(&nft.common.token_address)? + .with_value(nft.common.token_id.to_string())?; let nft_item = NftListTable::from_nft(&nft)?; nft_table @@ -449,7 +449,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) .with_value(chain.to_string())? - .with_value(&tx.transaction_hash)?; + .with_value(&tx.common.transaction_hash)?; let item = NftTxHistoryTable::from_tx_history(&tx)?; table.replace_item_by_unique_multi_index(index_keys, &item).await?; @@ -472,7 +472,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { let index_keys = MultiIndex::new(NftTxHistoryTable::CHAIN_TX_HASH_INDEX) .with_value(chain.to_string())? - .with_value(&tx.transaction_hash)?; + .with_value(&tx.common.transaction_hash)?; let item = NftTxHistoryTable::from_tx_history(&tx)?; table.replace_item_by_unique_multi_index(index_keys, &item).await?; @@ -571,10 +571,10 @@ impl NftListTable { fn from_nft(nft: &Nft) -> WasmNftCacheResult { let details_json = json::to_value(nft).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftListTable { - token_address: nft.token_address.clone(), - token_id: nft.token_id.to_string(), + token_address: nft.common.token_address.clone(), + token_id: nft.common.token_id.to_string(), chain: nft.chain.to_string(), - amount: nft.amount.to_string(), + amount: nft.common.amount.to_string(), block_number: BeBigUint::from(nft.block_number), contract_type: nft.contract_type, details_json, @@ -629,15 +629,15 @@ impl NftTxHistoryTable { fn from_tx_history(tx: &NftTransferHistory) -> WasmNftCacheResult { let details_json = json::to_value(tx).map_to_mm(|e| WasmNftCacheError::ErrorSerializing(e.to_string()))?; Ok(NftTxHistoryTable { - transaction_hash: tx.transaction_hash.clone(), + transaction_hash: tx.common.transaction_hash.clone(), chain: tx.chain.to_string(), block_number: BeBigUint::from(tx.block_number), block_timestamp: BeBigUint::from(tx.block_timestamp), contract_type: tx.contract_type, - token_address: tx.token_address.clone(), - token_id: tx.token_id.to_string(), + token_address: tx.common.token_address.clone(), + token_id: tx.common.token_id.to_string(), status: tx.status, - amount: tx.amount.to_string(), + amount: tx.common.amount.to_string(), token_uri: tx.token_uri.clone(), collection_name: tx.collection_name.clone(), image_url: tx.image_url.clone(), From 221c8e4cd77313be5fb369fe60c6946775caf8ec Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 6 Jul 2023 13:54:46 +0700 Subject: [PATCH 111/112] use `build_nft_from_moralis` --- mm2src/coins/nft.rs | 77 +++++++++++++++++---------------------------- 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index 4371c13571..cb31379b45 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -221,30 +221,7 @@ async fn get_moralis_nft_list(ctx: &MmArc, chain: &Chain, url: &Url) -> MmResult Some(contract_type) => contract_type, None => continue, }; - let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; - let nft = Nft { - common: NftCommon { - token_address: nft_moralis.common.token_address, - token_id: nft_moralis.common.token_id, - amount: nft_moralis.common.amount, - owner_of: nft_moralis.common.owner_of, - token_hash: nft_moralis.common.token_hash, - collection_name: nft_moralis.common.collection_name, - symbol: nft_moralis.common.symbol, - token_uri, - metadata: nft_moralis.common.metadata, - last_token_uri_sync: nft_moralis.common.last_token_uri_sync, - last_metadata_sync: nft_moralis.common.last_metadata_sync, - minter_address: nft_moralis.common.minter_address, - possible_spam: nft_moralis.common.possible_spam, - }, - chain: *chain, - block_number_minted: nft_moralis.block_number_minted.map(|v| v.0), - block_number: *nft_moralis.block_number, - contract_type, - uri_meta, - }; + let nft = build_nft_from_moralis(chain, nft_moralis, contract_type).await; // collect NFTs from the page res_list.push(nft); } @@ -377,30 +354,7 @@ async fn get_moralis_metadata( Some(contract_type) => contract_type, None => return MmError::err(GetNftInfoError::ContractTypeIsNull), }; - let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); - let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; - let nft_metadata = Nft { - common: NftCommon { - token_address: nft_moralis.common.token_address, - token_id: nft_moralis.common.token_id, - amount: nft_moralis.common.amount, - owner_of: nft_moralis.common.owner_of, - token_hash: nft_moralis.common.token_hash, - collection_name: nft_moralis.common.collection_name, - symbol: nft_moralis.common.symbol, - token_uri, - metadata: nft_moralis.common.metadata, - last_token_uri_sync: nft_moralis.common.last_token_uri_sync, - last_metadata_sync: nft_moralis.common.last_metadata_sync, - minter_address: nft_moralis.common.minter_address, - possible_spam: nft_moralis.common.possible_spam, - }, - chain: *chain, - block_number_minted: nft_moralis.block_number_minted.map(|v| v.0), - block_number: *nft_moralis.block_number, - contract_type, - uri_meta, - }; + let nft_metadata = build_nft_from_moralis(chain, nft_moralis, contract_type).await; Ok(nft_metadata) } @@ -863,3 +817,30 @@ fn check_spam_and_redact_metadata_field(metadata: &mut serde_json::Map false, } } + +async fn build_nft_from_moralis(chain: &Chain, nft_moralis: NftFromMoralis, contract_type: ContractType) -> Nft { + let token_uri = check_moralis_ipfs_bafy(nft_moralis.common.token_uri.as_deref()); + let uri_meta = get_uri_meta(token_uri.as_deref(), nft_moralis.common.metadata.as_deref()).await; + Nft { + common: NftCommon { + token_address: nft_moralis.common.token_address, + token_id: nft_moralis.common.token_id, + amount: nft_moralis.common.amount, + owner_of: nft_moralis.common.owner_of, + token_hash: nft_moralis.common.token_hash, + collection_name: nft_moralis.common.collection_name, + symbol: nft_moralis.common.symbol, + token_uri, + metadata: nft_moralis.common.metadata, + last_token_uri_sync: nft_moralis.common.last_token_uri_sync, + last_metadata_sync: nft_moralis.common.last_metadata_sync, + minter_address: nft_moralis.common.minter_address, + possible_spam: nft_moralis.common.possible_spam, + }, + chain: *chain, + block_number_minted: nft_moralis.block_number_minted.map(|v| v.0), + block_number: *nft_moralis.block_number, + contract_type, + uri_meta, + } +} From 5b0503318e3063e884988e95358a4d35f0b84fae Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 6 Jul 2023 19:16:39 +0700 Subject: [PATCH 112/112] add comments for cursor in wasm_storage, add regex to check links --- Cargo.lock | 1 + mm2src/coins/Cargo.toml | 1 + mm2src/coins/nft.rs | 52 +++++++++++-------- mm2src/coins/nft/nft_errors.rs | 16 +++++- mm2src/coins/nft/nft_tests.rs | 21 +++++++- mm2src/coins/nft/storage/wasm/wasm_storage.rs | 8 +++ 6 files changed, 76 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6c3c30dd35..eba3d3c01d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1042,6 +1042,7 @@ dependencies = [ "prost-build", "protobuf", "rand 0.7.3", + "regex", "rlp", "rmp-serde", "rpc", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index 7bdb8be0a3..72255359b3 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -76,6 +76,7 @@ primitives = { path = "../mm2_bitcoin/primitives" } prost = "0.10" protobuf = "2.20" rand = { version = "0.7", features = ["std", "small_rng"] } +regex = "1" rlp = { version = "0.5" } rmp-serde = "0.14.3" rpc = { path = "../mm2_bitcoin/rpc" } diff --git a/mm2src/coins/nft.rs b/mm2src/coins/nft.rs index cb31379b45..be05c7c076 100644 --- a/mm2src/coins/nft.rs +++ b/mm2src/coins/nft.rs @@ -15,12 +15,14 @@ use nft_structs::{Chain, ContractType, ConvertChain, Nft, NftFromMoralis, NftLis TransactionNftDetails, UpdateNftReq, WithdrawNftReq}; use crate::eth::{get_eth_address, withdraw_erc1155, withdraw_erc721}; +use crate::nft::nft_errors::ProtectFromSpamError; use crate::nft::nft_structs::{NftCommon, NftTransferCommon, RefreshMetadataReq, TransferStatus, TxMeta, UriMeta}; use crate::nft::storage::{NftListStorageOps, NftStorageBuilder, NftTxHistoryStorageOps}; use common::{parse_rfc3339_to_timestamp, APPLICATION_JSON}; use http::header::ACCEPT; use mm2_err_handle::map_to_mm::MapToMmResult; use mm2_number::BigDecimal; +use regex::Regex; use serde_json::Value as Json; use std::cmp::Ordering; @@ -89,7 +91,7 @@ pub async fn get_nft_transfers(ctx: MmArc, req: NftTransfersReq) -> MmResult bool { - text.contains("http://") || text.contains("https://") || text.contains("ftp://") || text.contains("file://") +fn contains_disallowed_url(text: &str) -> Result { + let url_regex = Regex::new( + r"(?:(?:https?|ftp|file|[^:\s]+:)/?|[^:\s]+:/|\b(?:[a-z\d]+\.))(?:(?:[^\s()<>]+|\((?:[^\s()<>]+|(?:\([^\s()<>]+\)))?\))+(?:\((?:[^\s()<>]+|(?:\(?:[^\s()<>]+\)))?\)|[^\s`!()\[\]{};:'.,<>?«»“”‘’]))?", + )?; + Ok(url_regex.is_match(text)) } -/// `check_and_redact_if_spam` checks if the text contains any links./// +/// `check_and_redact_if_spam` checks if the text contains any links. /// It doesn't matter if the link is valid or not, as this is a spam check. /// If text contains some link, then it is a spam. -fn check_and_redact_if_spam(text: &mut Option) -> bool { +fn check_and_redact_if_spam(text: &mut Option) -> Result { match text { - Some(s) if contains_disallowed_scheme(s) => { + Some(s) if contains_disallowed_url(s)? => { *text = Some("URL redacted for user protection".to_string()); - true + Ok(true) }, - _ => false, + _ => Ok(false), } } @@ -757,23 +762,25 @@ fn check_and_redact_if_spam(text: &mut Option) -> bool { /// /// `collection_name` and `token_name` in `NftTransferHistory` shouldn't contain any links, /// they must be just an arbitrary text, which represents NFT names. -fn protect_from_history_spam(tx: &mut NftTransferHistory) { - let collection_name_spam = check_and_redact_if_spam(&mut tx.collection_name); - let token_name_spam = check_and_redact_if_spam(&mut tx.token_name); +fn protect_from_history_spam(tx: &mut NftTransferHistory) -> MmResult<(), ProtectFromSpamError> { + let collection_name_spam = check_and_redact_if_spam(&mut tx.collection_name)?; + let token_name_spam = check_and_redact_if_spam(&mut tx.token_name)?; if collection_name_spam || token_name_spam { tx.common.possible_spam = true; } + Ok(()) } + /// `protect_from_nft_spam` function checks and redact spam in `Nft`. /// /// `collection_name` and `token_name` in `Nft` shouldn't contain any links, /// they must be just an arbitrary text, which represents NFT names. /// `symbol` also must be a text or sign that represents a symbol. -fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { - let collection_name_spam = check_and_redact_if_spam(&mut nft.common.collection_name); - let symbol_spam = check_and_redact_if_spam(&mut nft.common.symbol); - let token_name_spam = check_and_redact_if_spam(&mut nft.uri_meta.token_name); +fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), ProtectFromSpamError> { + let collection_name_spam = check_and_redact_if_spam(&mut nft.common.collection_name)?; + let symbol_spam = check_and_redact_if_spam(&mut nft.common.symbol)?; + let token_name_spam = check_and_redact_if_spam(&mut nft.uri_meta.token_name)?; let meta_spam = check_nft_metadata_for_spam(nft)?; if collection_name_spam || symbol_spam || token_name_spam || meta_spam { @@ -784,14 +791,14 @@ fn protect_from_nft_spam(nft: &mut Nft) -> MmResult<(), serde_json::Error> { /// `check_nft_metadata_for_spam` function checks and redact spam in `metadata` field from `Nft`. /// /// **note:** `token_name` is usually called `name` in `metadata`. -fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { +fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult { if let Some(Ok(mut metadata)) = nft .common .metadata .as_ref() .map(|t| serde_json::from_str::>(t)) { - if check_spam_and_redact_metadata_field(&mut metadata, "name") { + if check_spam_and_redact_metadata_field(&mut metadata, "name")? { nft.common.metadata = Some(serde_json::to_string(&metadata)?); return Ok(true); } @@ -805,16 +812,19 @@ fn check_nft_metadata_for_spam(nft: &mut Nft) -> MmResult, field: &str) -> bool { +fn check_spam_and_redact_metadata_field( + metadata: &mut serde_json::Map, + field: &str, +) -> MmResult { match metadata.get(field).and_then(|v| v.as_str()) { - Some(text) if contains_disallowed_scheme(text) => { + Some(text) if contains_disallowed_url(text)? => { metadata.insert( field.to_string(), serde_json::Value::String("URL redacted for user protection".to_string()), ); - true + Ok(true) }, - _ => false, + _ => Ok(false), } } diff --git a/mm2src/coins/nft/nft_errors.rs b/mm2src/coins/nft/nft_errors.rs index c2f074d02b..62ddbefef8 100644 --- a/mm2src/coins/nft/nft_errors.rs +++ b/mm2src/coins/nft/nft_errors.rs @@ -36,6 +36,7 @@ pub enum GetNftInfoError { ParseRfc3339Err(ParseRfc3339Err), #[display(fmt = "The contract type is required and should not be null.")] ContractTypeIsNull, + ProtectFromSpamError(ProtectFromSpamError), } impl From for WithdrawError { @@ -98,6 +99,10 @@ impl From for GetNftInfoError { fn from(e: ParseRfc3339Err) -> Self { GetNftInfoError::ParseRfc3339Err(e) } } +impl From for GetNftInfoError { + fn from(e: ProtectFromSpamError) -> Self { GetNftInfoError::ProtectFromSpamError(e) } +} + impl HttpStatusCode for GetNftInfoError { fn status_code(&self) -> StatusCode { match self { @@ -108,7 +113,8 @@ impl HttpStatusCode for GetNftInfoError { | GetNftInfoError::Internal(_) | GetNftInfoError::GetEthAddressError(_) | GetNftInfoError::TokenNotFoundInWallet { .. } - | GetNftInfoError::DbError(_) => StatusCode::INTERNAL_SERVER_ERROR, + | GetNftInfoError::DbError(_) + | GetNftInfoError::ProtectFromSpamError(_) => StatusCode::INTERNAL_SERVER_ERROR, } } } @@ -224,3 +230,11 @@ impl From for GetInfoFromUriError { } } } + +#[derive(Clone, Debug, Deserialize, Display, EnumFromStringify, PartialEq, Serialize)] +pub enum ProtectFromSpamError { + #[from_stringify("regex::Error")] + RegexError(String), + #[from_stringify("serde_json::Error")] + SerdeError(String), +} diff --git a/mm2src/coins/nft/nft_tests.rs b/mm2src/coins/nft/nft_tests.rs index 94c3abe3e2..6d3c0ea46d 100644 --- a/mm2src/coins/nft/nft_tests.rs +++ b/mm2src/coins/nft/nft_tests.rs @@ -31,10 +31,29 @@ mod native_tests { #[test] fn test_check_for_spam() { let mut spam_text = Some("https://arweave.net".to_string()); - assert!(check_and_redact_if_spam(&mut spam_text)); + assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); let url_redacted = "URL redacted for user protection"; assert_eq!(url_redacted, spam_text.unwrap()); + let mut spam_text = Some("ftp://123path ".to_string()); + assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some("/192.168.1.1/some.example.org?type=A".to_string()); + assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut spam_text = Some(r"C:\Users\path\".to_string()); + assert!(check_and_redact_if_spam(&mut spam_text).unwrap()); + let url_redacted = "URL redacted for user protection"; + assert_eq!(url_redacted, spam_text.unwrap()); + + let mut valid_text = Some("Hello my name is NFT (The best ever!)".to_string()); + assert!(!check_and_redact_if_spam(&mut valid_text).unwrap()); + assert_eq!("Hello my name is NFT (The best ever!)", valid_text.unwrap()); + let mut nft = nft(); assert!(check_nft_metadata_for_spam(&mut nft).unwrap()); let meta_redacted = "{\"name\":\"URL redacted for user protection\",\"image\":\"https://tikimetadata.s3.amazonaws.com/tiki_box.png\"}"; diff --git a/mm2src/coins/nft/storage/wasm/wasm_storage.rs b/mm2src/coins/nft/storage/wasm/wasm_storage.rs index 89237f1ba2..eb8c22e17b 100644 --- a/mm2src/coins/nft/storage/wasm/wasm_storage.rs +++ b/mm2src/coins/nft/storage/wasm/wasm_storage.rs @@ -513,6 +513,7 @@ impl NftTxHistoryStorageOps for IndexedDbNftStorage { } } +/// `get_last_block_from_table` function returns the highest block in the table related to certain blockchain type. async fn get_last_block_from_table( chain: &Chain, table: DbTable<'_, impl TableSignature + BlockNumberTable>, @@ -522,11 +523,18 @@ async fn get_last_block_from_table( .cursor_builder() .only("chain", chain.to_string()) .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + // Sets lower and upper bounds for block_number field .bound("block_number", BeBigUint::from(0u64), BeBigUint::from(u64::MAX)) + // Cursor returns values from the lowest to highest key indexes. + // But we need to get the most highest block_number, so reverse the cursor direction. .reverse() + // Opens a cursor by the specified index. + // In get_last_block_from_table case it is CHAIN_BLOCK_NUMBER_INDEX, as we need to search block_number for specific chain. .open_cursor(cursor) .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))? + // Advances the iterator and returns the next value. + // Please note that the items are sorted by the index keys. .next() .await .map_err(|e| WasmNftCacheError::GetLastNftBlockError(e.to_string()))?;