Skip to content

Commit caf9ef3

Browse files
dangershonydangershony
authored
Reading history from db instead of loading all entries in memory (#186)
* Reading history from db instead of loading all entries in memory * limit to last 100 entries * Fix all tests * add wallet history tests
1 parent f2debd7 commit caf9ef3

34 files changed

Lines changed: 602 additions & 204 deletions

‎src/Features/Blockcore.Features.Wallet/Api/Controllers/WalletModelBuilder.cs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
using System.Linq;
44
using Blockcore.Features.Wallet.Api.Models;
55
using Blockcore.Features.Wallet.Database;
6+
using Blockcore.Features.Wallet.Exceptions;
67
using Blockcore.Features.Wallet.Helpers;
78
using Blockcore.Features.Wallet.Interfaces;
89
using Blockcore.Features.Wallet.Types;
@@ -56,6 +57,104 @@ public static Mnemonic GenerateMnemonic(string language = "English", int wordCou
5657
return mnemonic;
5758
}
5859

60+
/// <summary>
61+
/// This method is an attempt to fetch history and its complex calculation directly from disk
62+
/// and avoid the need to fetch the entire history to memory
63+
/// </summary>
64+
public static WalletHistoryModel GetHistorySlim(IWalletManager walletManager, Network network, WalletHistoryRequest request)
65+
{
66+
if (!string.IsNullOrEmpty(request.Address))
67+
{
68+
throw new NotImplementedException("Search by address not implemented");
69+
}
70+
71+
if (!string.IsNullOrEmpty(request.SearchQuery))
72+
{
73+
throw new NotImplementedException("Search by SearchQuery not implemented");
74+
}
75+
76+
var model = new WalletHistoryModel();
77+
78+
// Get a list of all the transactions found in an account (or in a wallet if no account is specified), with the addresses associated with them.
79+
IEnumerable<AccountHistorySlim> accountsHistory = walletManager.GetHistorySlim(request.WalletName, request.AccountName, skip: request.Skip ?? 0, take: request.Take ?? int.MaxValue);
80+
81+
foreach (AccountHistorySlim accountHistory in accountsHistory)
82+
{
83+
var transactionItems = new List<TransactionItemModel>();
84+
85+
foreach (FlatHistorySlim item in accountHistory.History)
86+
{
87+
var modelItem = new TransactionItemModel
88+
{
89+
Type = item.Transaction.IsSent ? TransactionItemType.Send : TransactionItemType.Received,
90+
ToAddress = item.Transaction.Address,
91+
Amount = item.Transaction.IsSent == false ? item.Transaction.Amount : Money.Zero,
92+
Id = item.Transaction.IsSent ? item.Transaction.SentTo : item.Transaction.OutPoint.Hash,
93+
Timestamp = item.Transaction.CreationTime,
94+
ConfirmedInBlock = item.Transaction.BlockHeight,
95+
BlockIndex = item.Transaction.BlockIndex
96+
};
97+
98+
if (item.Transaction.IsSent == true) // handle send entries
99+
{
100+
// First we look for staking transaction as they require special attention.
101+
// A staking transaction spends one of our inputs into 2 outputs or more, paid to the same address.
102+
if ((item.Transaction.IsCoinStake ?? false == true))
103+
{
104+
if (item.Transaction.IsSent == true)
105+
{
106+
modelItem.Type = TransactionItemType.Staked;
107+
var amount = item.Transaction.SentPayments.Sum(p => p.Amount);
108+
modelItem.Amount = amount - item.Transaction.Amount;
109+
}
110+
else
111+
{
112+
// We don't show in history transactions that are outputs of staking transactions.
113+
continue;
114+
}
115+
}
116+
else
117+
{
118+
if (item.Transaction.SentPayments.All(a => a.PayToSelf == true))
119+
{
120+
// if all outputs are to ourself
121+
// we don't show that in history
122+
continue;
123+
}
124+
125+
modelItem.Amount = item.Transaction.SentPayments.Where(x => x.PayToSelf == false).Sum(p => p.Amount);
126+
}
127+
}
128+
else // handle receive entries
129+
{
130+
if (item.Address.IsChangeAddress())
131+
{
132+
// we don't display transactions sent to self
133+
continue;
134+
}
135+
136+
if (item.Transaction.IsCoinStake.HasValue && item.Transaction.IsCoinStake.Value == true)
137+
{
138+
// We don't show in history transactions that are outputs of staking transactions.
139+
continue;
140+
}
141+
}
142+
143+
transactionItems.Add(modelItem);
144+
}
145+
146+
model.AccountsHistoryModel.Add(new AccountHistoryModel
147+
{
148+
TransactionsHistory = transactionItems,
149+
Name = accountHistory.Account.Name,
150+
CoinType = network.Consensus.CoinType,
151+
HdPath = accountHistory.Account.HdPath
152+
});
153+
}
154+
155+
return model;
156+
}
157+
59158
public static WalletHistoryModel GetHistory(IWalletManager walletManager, Network network, WalletHistoryRequest request)
60159
{
61160
var model = new WalletHistoryModel();

‎src/Features/Blockcore.Features.Wallet/Database/IWalletStore.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,5 +25,7 @@ public interface IWalletStore
2525
WalletBalanceResult GetBalanceForAddress(string address, bool excludeColdStake);
2626

2727
WalletBalanceResult GetBalanceForAccount(int accountIndex, bool excludeColdStake);
28+
29+
IEnumerable<WalletHistoryData> GetAccountHistory(int accountIndex, bool excludeColdStake, int skip = 0, int take = 100);
2830
}
2931
}

‎src/Features/Blockcore.Features.Wallet/Database/SpendingDetails.cs‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using NBitcoin;
4+
5+
namespace Blockcore.Features.Wallet.Database
6+
{
7+
public class PaymentDetails
8+
{
9+
public Script DestinationScriptPubKey { get; set; }
10+
11+
public string DestinationAddress { get; set; }
12+
13+
public int? OutputIndex { get; set; }
14+
15+
public Money Amount { get; set; }
16+
17+
public bool? PayToSelf { get; set; }
18+
}
19+
20+
public class SpendingDetails
21+
{
22+
public SpendingDetails()
23+
{
24+
this.Payments = new List<PaymentDetails>();
25+
}
26+
27+
public uint256 TransactionId { get; set; }
28+
29+
public ICollection<PaymentDetails> Payments { get; set; }
30+
31+
public int? BlockHeight { get; set; }
32+
33+
public int? BlockIndex { get; set; }
34+
35+
public bool? IsCoinStake { get; set; }
36+
37+
public DateTimeOffset CreationTime { get; set; }
38+
39+
public string Hex { get; set; }
40+
41+
public bool IsSpentConfirmed()
42+
{
43+
return this.BlockHeight != null;
44+
}
45+
}
46+
}

‎src/Features/Blockcore.Features.Wallet/Database/WalletData.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
using System;
12
using System.Collections.Generic;
23
using Blockcore.Utilities;
34
using LiteDB;
@@ -24,4 +25,32 @@ public class WalletData
2425

2526
public ICollection<uint256> BlockLocator { get; set; }
2627
}
28+
29+
public class WalletHistoryData
30+
{
31+
public bool IsSent { get; set; }
32+
public ICollection<WalletHistoryData> ReceivedOutputs { get; set; }
33+
34+
public ICollection<WalletHistoryPaymentData> SentPayments { get; set; }
35+
36+
public OutPoint OutPoint { get; set; }
37+
public uint256 SentTo { get; set; }
38+
public string Address { get; set; }
39+
public Money Amount { get; set; }
40+
public bool? IsCoinBase { get; set; }
41+
public bool? IsCoinStake { get; set; }
42+
public bool? IsColdCoinStake { get; set; }
43+
public int? BlockHeight { get; set; }
44+
public int? BlockIndex { get; set; }
45+
public DateTimeOffset CreationTime { get; set; }
46+
public Script ScriptPubKey { get; set; }
47+
}
48+
49+
public class WalletHistoryPaymentData
50+
{
51+
public string DestinationAddress { get; set; }
52+
53+
public Money Amount { get; set; }
54+
public bool? PayToSelf { get; set; }
55+
}
2756
}

‎src/Features/Blockcore.Features.Wallet/Database/WalletStore.cs‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using Blockcore.Configuration;
66
using Blockcore.Features.Wallet.Exceptions;
77
using Blockcore.Utilities;
8+
using DBreeze.Utils;
89
using LiteDB;
910
using NBitcoin;
1011
using NBitcoin.DataEncoders;
@@ -108,6 +109,104 @@ public void InsertOrUpdate(TransactionOutputData item)
108109
this.trxCol.Upsert(item);
109110
}
110111

112+
public IEnumerable<WalletHistoryData> GetAccountHistory(int accountIndex, bool excludeColdStake, int skip = 0, int take = 100)
113+
{
114+
// The result of this method is not guaranteed to be the length
115+
// of the 'take' param. In case some of the inputs we have are
116+
// in the same trx they will be grouped in to a single entry.
117+
118+
var historySpent = this.trxCol.Query()
119+
.Where(x => x.AccountIndex == accountIndex)
120+
.Where(x => x.SpendingDetails != null)
121+
.Where(x => excludeColdStake ? (x.IsColdCoinStake != true) : true)
122+
.OrderByDescending(x => x.SpendingDetails.CreationTime)
123+
.Skip(skip)
124+
.Limit(take)
125+
.ToList();
126+
127+
var historyUnspent = this.trxCol.Query()
128+
.Where(x => x.AccountIndex == accountIndex)
129+
.Where(x => excludeColdStake ? (x.IsColdCoinStake != true) : true)
130+
.OrderByDescending(x => x.CreationTime)
131+
.Skip(skip)
132+
.Limit(take)
133+
.ToList();
134+
135+
var items = new List<WalletHistoryData>();
136+
137+
items.AddRange(historySpent
138+
.GroupBy(g => g.SpendingDetails.TransactionId)
139+
.Select(s =>
140+
{
141+
var x = s.First();
142+
143+
return new WalletHistoryData
144+
{
145+
IsSent = true,
146+
SentTo = x.SpendingDetails.TransactionId,
147+
IsCoinStake = x.SpendingDetails.IsCoinStake,
148+
CreationTime = x.SpendingDetails.CreationTime,
149+
BlockHeight = x.SpendingDetails.BlockHeight,
150+
BlockIndex = x.SpendingDetails.BlockIndex,
151+
SentPayments = x.SpendingDetails.Payments?.Select(p => new WalletHistoryPaymentData
152+
{
153+
Amount = p.Amount,
154+
PayToSelf = p.PayToSelf,
155+
DestinationAddress = p.DestinationAddress
156+
}).ToList(),
157+
158+
// when spent the amount represents the
159+
// input that was spent not the output
160+
Amount = x.Amount
161+
};
162+
}));
163+
164+
items.AddRange(historyUnspent
165+
.GroupBy(g => g.Id)
166+
.Select(s =>
167+
{
168+
var x = s.First();
169+
170+
var ret = new WalletHistoryData
171+
{
172+
IsSent = false,
173+
OutPoint = x.OutPoint,
174+
BlockHeight = x.BlockHeight,
175+
BlockIndex = x.BlockIndex,
176+
IsCoinStake = x.IsCoinStake,
177+
CreationTime = x.CreationTime,
178+
ScriptPubKey = x.ScriptPubKey,
179+
Address = x.Address,
180+
Amount = x.Amount,
181+
IsCoinBase = x.IsCoinBase,
182+
IsColdCoinStake = x.IsColdCoinStake,
183+
};
184+
185+
if (s.Count() > 1)
186+
{
187+
ret.Amount = s.Sum(b => b.Amount);
188+
ret.ReceivedOutputs = s.Select(b => new WalletHistoryData
189+
{
190+
IsSent = false,
191+
OutPoint = b.OutPoint,
192+
BlockHeight = b.BlockHeight,
193+
BlockIndex = b.BlockIndex,
194+
IsCoinStake = b.IsCoinStake,
195+
CreationTime = b.CreationTime,
196+
ScriptPubKey = b.ScriptPubKey,
197+
Address = b.Address,
198+
Amount = b.Amount,
199+
IsCoinBase = b.IsCoinBase,
200+
IsColdCoinStake = b.IsColdCoinStake,
201+
}).ToList();
202+
}
203+
204+
return ret;
205+
}));
206+
207+
return items.OrderByDescending(x => x.CreationTime).ThenBy(x => x.BlockIndex);
208+
}
209+
111210
public IEnumerable<TransactionOutputData> GetForAddress(string address)
112211
{
113212
var trxs = this.trxCol.Find(Query.EQ("Address", new BsonValue(address)));

‎src/Features/Blockcore.Features.Wallet/Interfaces/IWalletManager.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -215,6 +215,27 @@ public interface IWalletManager
215215
/// <returns>The history for this account.</returns>
216216
AccountHistory GetHistory(Types.Wallet wallet, HdAccount account);
217217

218+
/// <summary>
219+
/// Gets the history of transactions contained in an account.
220+
/// If no account name is specified, history will be returned for all accounts in the wallet.
221+
/// </summary>
222+
/// <param name="walletName">The wallet name.</param>
223+
/// <param name="accountName">The account name.</param>
224+
/// <param name="skip">Items to skip.</param>
225+
/// <param name="take">Items to take.</param>
226+
/// <returns>Collection of address history and transaction pairs.</returns>
227+
IEnumerable<AccountHistorySlim> GetHistorySlim(string walletName, string accountName = null, int skip = 0, int take = 100);
228+
229+
/// <summary>
230+
/// Gets the history of the transactions in addresses contained in this account.
231+
/// </summary>
232+
/// <param name="wallet">The wallet instance.</param>
233+
/// <param name="account">The account for which to get history.</param>
234+
/// <param name="skip">Items to skip.</param>
235+
/// <param name="take">Items to take.</param>
236+
/// <returns>The history for this account.</returns>
237+
AccountHistorySlim GetHistorySlim(Types.Wallet wallet, HdAccount account, int skip = 0, int take = 100);
238+
218239
/// <summary>
219240
/// Gets the balance of transactions contained in an account.
220241
/// If no account name is specified, balances will be returned for all accounts in the wallet.

‎src/Features/Blockcore.Features.Wallet/Types/FlatHistory.cs‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,33 @@ public class FlatHistory
3131
/// </summary>
3232
public TransactionOutputData Transaction { get; set; }
3333
}
34+
35+
public class AccountHistorySlim
36+
{
37+
/// <summary>
38+
/// The account for which the history is retrieved.
39+
/// </summary>
40+
public HdAccount Account { get; set; }
41+
42+
/// <summary>
43+
/// The collection of history items.
44+
/// </summary>
45+
public IEnumerable<FlatHistorySlim> History { get; set; }
46+
}
47+
48+
/// <summary>
49+
/// A class that represents a flat view of the wallets history.
50+
/// </summary>
51+
public class FlatHistorySlim
52+
{
53+
/// <summary>
54+
/// The address associated with this UTXO.
55+
/// </summary>
56+
public HdAddress Address { get; set; }
57+
58+
/// <summary>
59+
/// The transaction representing the UTXO.
60+
/// </summary>
61+
public WalletHistoryData Transaction { get; set; }
62+
}
3463
}

0 commit comments

Comments
 (0)