206 changes: 203 additions & 3 deletions src/txdb.cpp
Expand Up @@ -13,6 +13,10 @@
#include <util.h>
#include <ui_interface.h>
#include <init.h>
#include <validation.h>

#include <script/standard.h>
#include <base58.h>

#include <stdint.h>

Expand All @@ -30,6 +34,7 @@ static const char DB_FLAG = 'F';
static const char DB_REINDEX_FLAG = 'R';
static const char DB_LAST_BLOCK = 'l';

static const char DB_LOADED_COINS = 'p';
namespace {

struct CoinEntry {
Expand All @@ -54,16 +59,28 @@ struct CoinEntry {

}

CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true)
CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize / 2, fMemory, fWipe, true) , loadedcoindb(GetDataDir() / "loadedcoins", nCacheSize / 2, false, false, true)
{
}

bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const {
return db.Read(CoinEntry(&outpoint), coin);
if (db.Read(CoinEntry(&outpoint), coin))
return true;

LoadedCoin loadedCoin;
if (GetLoadedCoin(outpoint.GetHash(), loadedCoin)) {
coin = loadedCoin.coin;
return true;
}

return false;
}

bool CCoinsViewDB::HaveCoin(const COutPoint &outpoint) const {
return db.Exists(CoinEntry(&outpoint));
if (db.Exists(CoinEntry(&outpoint)))
return true;
else
return HaveLoadedCoin(outpoint.GetHash());
}

uint256 CCoinsViewDB::GetBestBlock() const {
Expand Down Expand Up @@ -147,6 +164,162 @@ size_t CCoinsViewDB::EstimateSize() const
return db.EstimateSize(DB_COIN, (char)(DB_COIN+1));
}

bool CCoinsViewDB::WriteLoadedCoinIndex(const std::vector<LoadedCoin>& vLoadedCoin)
{
CDBBatch batch(loadedcoindb);

for (const LoadedCoin& c : vLoadedCoin) {
std::pair<char, uint256> key = std::make_pair(DB_LOADED_COINS, c.out.GetHash());
batch.Write(key, c);
}

return loadedcoindb.WriteBatch(batch);
}

bool CCoinsViewDB::GetLoadedCoin(const uint256& hashOutPoint, LoadedCoin& coinOut) const
{
std::unique_ptr<CDBIterator> pcursor(const_cast<CDBWrapper&>(loadedcoindb).NewIterator());
pcursor->Seek(std::make_pair(DB_LOADED_COINS, hashOutPoint));
if (pcursor->Valid()) {
try {
std::pair<char, uint256> key;
pcursor->GetKey(key);

if (key.second == hashOutPoint) {
pcursor->GetValue(coinOut);
return true;
}
} catch (const std::exception& e) {
error("%s: %s", __func__, e.what());
}
}
return false;
}

bool CCoinsViewDB::HaveLoadedCoin(const uint256& hashOutPoint) const
{
LoadedCoin coin;
return GetLoadedCoin(hashOutPoint, coin);
}

bool CCoinsViewDB::ReadLoadedCoins()
{
fs::path path = GetDataDir() / "loaded_coins.dat";
CAutoFile filein(fsbridge::fopen(path, "r"), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
return false;
}

// TODO log this
uint64_t fileSize = fs::file_size(path);
uint64_t dataSize = fileSize - (sizeof(int) * 3);

int read = 0;
std::vector<LoadedCoin> vLoadedCoin;
try {
int nVersionRequired, nVersionThatWrote;
filein >> nVersionRequired;
filein >> nVersionThatWrote;
if (nVersionRequired > CLIENT_VERSION) {
return false;
}

int count = 0;
filein >> count;
for (int i = 0; i < count; i++) {
if (i % 2000000 == 0) {
// Write a batch of loaded coins to the index.
// Batches are 2000000 coins each, which is around 200MB.
WriteLoadedCoinIndex(vLoadedCoin);
vLoadedCoin.clear();
}

LoadedCoin loadedCoin;
filein >> loadedCoin;
vLoadedCoin.push_back(loadedCoin);
read++;
}
// Write final batch (or up to 2000000 coins will be skipped)
WriteLoadedCoinIndex(vLoadedCoin);
}
catch (const std::exception& e) {
// TODO log this
// std::cout << "Exception: " << e.what() << std::endl;
return false;
}

// TODO log how many coins were read

return true;
}

std::vector<LoadedCoin> CCoinsViewDB::ReadMyLoadedCoins()
{
std::vector<LoadedCoin> vLoadedCoin;

fs::path path = GetDataDir() / "my_loaded_coins.dat";
CAutoFile filein(fsbridge::fopen(path, "r"), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
return vLoadedCoin;
}

uint64_t fileSize = fs::file_size(path);
uint64_t dataSize = fileSize - (sizeof(int) * 3);

try {
int nVersionRequired, nVersionThatWrote;
filein >> nVersionRequired;
filein >> nVersionThatWrote;
if (nVersionRequired > CLIENT_VERSION) {
return vLoadedCoin;
}

int count = 0;
filein >> count;
for (int i = 0; i < count; i++) {
LoadedCoin loadedCoin;
filein >> loadedCoin;
vLoadedCoin.push_back(loadedCoin);
}
}
catch (const std::exception& e) {
// TODO log this
// std::cout << "Exception: " << e.what() << std::endl;
return vLoadedCoin;
}

return vLoadedCoin;
}

void CCoinsViewDB::WriteMyLoadedCoins(const std::vector<LoadedCoin>& vLoadedCoin)
{
if (!vLoadedCoin.size())
return;
int count = vLoadedCoin.size();

// Write the coins
fs::path path = GetDataDir() / "my_loaded_coins.dat";
CAutoFile fileout(fsbridge::fopen(path, "w"), SER_DISK, CLIENT_VERSION);
if (fileout.IsNull()) {
return;
}

try {
fileout << 210000; // version required to read: 0.21.00 or later
fileout << CLIENT_VERSION; // version that wrote the file
fileout << count; // Number of coins in file

for (const LoadedCoin& c : vLoadedCoin) {
fileout << c;
}
}
catch (const std::exception& e) {
// TODO log this
// std::cout << "ML Exception: " << e.what() << std::endl;
return;
}
}

CBlockTreeDB::CBlockTreeDB(size_t nCacheSize, bool fMemory, bool fWipe) : CDBWrapper(GetDataDir() / "blocks" / "index", nCacheSize, fMemory, fWipe) {
}

Expand Down Expand Up @@ -188,6 +361,13 @@ CCoinsViewCursor *CCoinsViewDB::Cursor() const
return i;
}

CCoinsViewLoadedCursor *CCoinsViewDB::LoadedCursor() const
{
CCoinsViewLoadedDBCursor *i = new CCoinsViewLoadedDBCursor(const_cast<CDBWrapper&>(loadedcoindb).NewIterator());
i->pcursor->Seek(DB_LOADED_COINS);
return i;
}

bool CCoinsViewDBCursor::GetKey(COutPoint &key) const
{
// Return cached key
Expand Down Expand Up @@ -224,6 +404,26 @@ void CCoinsViewDBCursor::Next()
}
}

bool CCoinsViewLoadedDBCursor::GetKey(std::pair<char, uint256>& key) const
{
return pcursor->GetKey(key);
}

bool CCoinsViewLoadedDBCursor::GetValue(LoadedCoin &coin) const
{
return pcursor->GetValue(coin);
}

bool CCoinsViewLoadedDBCursor::Valid() const
{
return pcursor->Valid();
}

void CCoinsViewLoadedDBCursor::Next()
{
pcursor->Next();
}

bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
CDBBatch batch(*this);
for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
Expand Down
39 changes: 38 additions & 1 deletion src/txdb.h
Expand Up @@ -17,12 +17,13 @@

class CBlockIndex;
class CCoinsViewDBCursor;
class CCoinsViewLoadedDBCursor;
class uint256;

//! No need to periodic flush if at least this much space still available.
static constexpr int MAX_BLOCK_COINSDB_USAGE = 10;
//! -dbcache default (MiB)
static const int64_t nDefaultDbCache = 450;
static const int64_t nDefaultDbCache = 650;
//! -dbbatchsize default (bytes)
static const int64_t nDefaultDbBatchSize = 16 << 20;
//! max. -dbcache (MiB)
Expand Down Expand Up @@ -68,6 +69,7 @@ class CCoinsViewDB final : public CCoinsView
{
protected:
CDBWrapper db;
CDBWrapper loadedcoindb;
public:
explicit CCoinsViewDB(size_t nCacheSize, bool fMemory = false, bool fWipe = false);

Expand All @@ -77,10 +79,25 @@ class CCoinsViewDB final : public CCoinsView
std::vector<uint256> GetHeadBlocks() const override;
bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) override;
CCoinsViewCursor *Cursor() const override;
CCoinsViewLoadedCursor *LoadedCursor() const;

//! Attempt to update from an older database format. Returns whether an error occurred.
bool Upgrade();
size_t EstimateSize() const override;

// Loaded coins functions

// LDB interaction
bool WriteLoadedCoinIndex(const std::vector<LoadedCoin>& vLoadedCoin);
bool GetLoadedCoin(const uint256& hashOutPoint, LoadedCoin& coinOut) const;
bool HaveLoadedCoin(const uint256& hashOutPoint) const;
void RescanLoadedCoins(std::vector<LoadedCoin>& vLoadedCoin) const;

// .dat file interaction
bool ReadLoadedCoins();
bool WriteLoadedCoins(); // Note: only used for created loaded_coins.dat
std::vector<LoadedCoin> ReadMyLoadedCoins();
void WriteMyLoadedCoins(const std::vector<LoadedCoin>& vLoadedCoin);
};

/** Specialization of CCoinsViewCursor to iterate over a CCoinsViewDB */
Expand All @@ -105,6 +122,26 @@ class CCoinsViewDBCursor: public CCoinsViewCursor
friend class CCoinsViewDB;
};

/** Specialization of CCoinsViewLoadedCursor to iterate over a CCoinsViewDB */
class CCoinsViewLoadedDBCursor: public CCoinsViewLoadedCursor
{
public:
~CCoinsViewLoadedDBCursor() {}

bool GetKey(std::pair<char, uint256>& key) const override;
bool GetValue(LoadedCoin& coin) const override;

bool Valid() const override;
void Next() override;

private:
CCoinsViewLoadedDBCursor(CDBIterator* pcursorIn):
CCoinsViewLoadedCursor(), pcursor(pcursorIn) {}
std::unique_ptr<CDBIterator> pcursor;

friend class CCoinsViewDB;
};

/** Access to the block database (blocks/index/) */
class CBlockTreeDB : public CDBWrapper
{
Expand Down
2 changes: 1 addition & 1 deletion src/validation.cpp
Expand Up @@ -52,7 +52,7 @@
#include <boost/thread.hpp>

#if defined(NDEBUG)
# error "Bitcoin cannot be compiled without assertions."
# error "DriveNet cannot be compiled without assertions."
#endif

#define MICRO 0.000001
Expand Down
6 changes: 6 additions & 0 deletions src/wallet/feebumper.cpp
Expand Up @@ -32,6 +32,12 @@ static int64_t CalculateMaximumSignedTxSize(const CTransaction &tx, const CWalle
// wallet, with a valid index into the vout array.
for (auto& input : tx.vin) {
const auto mi = wallet->mapWallet.find(input.prevout.hash);

// TODO If the wallet cannot find the wallet tx, we might be spending a
// loaded coin input. Skip cointing it for the fee bumper.
if (mi == wallet->mapWallet.end())
continue;

assert(mi != wallet->mapWallet.end() && input.prevout.n < mi->second.tx->vout.size());
vCoins.emplace_back(CInputCoin(&(mi->second), input.prevout.n));
}
Expand Down
35 changes: 34 additions & 1 deletion src/wallet/rpcdump.cpp
Expand Up @@ -4,6 +4,7 @@

#include <base58.h>
#include <chain.h>
#include <coins.h>
#include <rpc/safemode.h>
#include <rpc/server.h>
#include <wallet/init.h>
Expand Down Expand Up @@ -62,7 +63,7 @@ std::string DecodeDumpString(const std::string &str) {
for (unsigned int pos = 0; pos < str.length(); pos++) {
unsigned char c = str[pos];
if (c == '%' && pos+2 < str.length()) {
c = (((str[pos+1]>>6)*9+((str[pos+1]-'0')&15)) << 4) |
c = (((str[pos+1]>>6)*9+((str[pos+1]-'0')&15)) << 4) |
((str[pos+2]>>6)*9+((str[pos+2]-'0')&15));
pos += 2;
}
Expand Down Expand Up @@ -184,6 +185,38 @@ UniValue importprivkey(const JSONRPCRequest& request)
pwallet->RescanFromTime(TIMESTAMP_MIN, reserver, true /* update */);
}

// And also rescan loaded coins and start tracking those
std::vector<LoadedCoin> vLoadedCoin;
{
LOCK(cs_main);
CCoinsViewLoadedCursor *i = pcoinsTip->LoadedCursor();

CWallet* const pwallet = GetWalletForJSONRPCRequest(request);
if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) {
return NullUniValue;
}

while (i->Valid()) {
LoadedCoin coin;
i->GetValue(coin);

if (pwallet->IsSpent(coin.out.hash, coin.out.n))
continue;

isminetype mine = pwallet->IsMine(coin.coin.out);
if (mine == ISMINE_NO) {
i->Next();
continue;
}

vLoadedCoin.push_back(coin);

i->Next();
}
}
if (vLoadedCoin.size())
pwallet->AddLoadedCoins(vLoadedCoin);

return NullUniValue;
}

Expand Down
42 changes: 42 additions & 0 deletions src/wallet/wallet.cpp
Expand Up @@ -11,6 +11,8 @@
#include <wallet/coincontrol.h>
#include <consensus/consensus.h>
#include <consensus/validation.h>
#include <coins.h>
#include <dbwrapper.h>
#include <fs.h>
#include <wallet/init.h>
#include <key.h>
Expand Down Expand Up @@ -2066,6 +2068,12 @@ CAmount CWallet::GetBalance() const
if (pcoin->IsTrusted())
nTotal += pcoin->GetAvailableCredit();
}
// Also count loaded coins
for (const LoadedCoin& c : vLoadedCoinCache) {
if (!IsSpent(c.out.hash, c.out.n)) {
nTotal += c.coin.out.nValue;
}
}
}

return nTotal;
Expand Down Expand Up @@ -2200,6 +2208,11 @@ CAmount CWallet::GetAvailableBalance(const CCoinControl* coinControl) const
balance += out.tx->tx->vout[out.i].nValue;
}
}
// Also count loaded coins
for (const LoadedCoin& c : vLoadedCoinCache) {
if (!IsSpent(c.out.hash, c.out.n))
balance += c.coin.out.nValue;
}
return balance;
}

Expand Down Expand Up @@ -2554,6 +2567,8 @@ bool CWallet::SelectCoinsMinConf(const CAmount& nTargetValue, const int nConfMin
bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAmount& nTargetValue, std::set<CInputCoin>& setCoinsRet, CAmount& nValueRet, const CCoinControl* coinControl) const
{
std::vector<COutput> vCoins(vAvailableCoins);
std::vector<LoadedCoin> vLoadedCoin;
vLoadedCoin = GetMyLoadedCoins();

// coin control -> return all selected outputs (we want all selected to go into the transaction for sure)
if (coinControl && coinControl->HasSelected() && !coinControl->fAllowOtherInputs)
Expand All @@ -2565,6 +2580,13 @@ bool CWallet::SelectCoins(const std::vector<COutput>& vAvailableCoins, const CAm
nValueRet += out.tx->tx->vout[out.i].nValue;
setCoinsRet.insert(CInputCoin(out.tx, out.i));
}
for (const LoadedCoin& c : vLoadedCoin)
{
if (IsSpent(c.out.hash, c.out.n))
continue;
nValueRet += c.coin.out.nValue;
setCoinsRet.insert(CInputCoin(c.out, c.coin.out));
}
return (nValueRet >= nTargetValue);
}

Expand Down Expand Up @@ -3251,6 +3273,13 @@ bool CWallet::CommitTransaction(CWalletTx& wtxNew, CReserveKey& reservekey, CCon
// Notify that old coins are spent
for (const CTxIn& txin : wtxNew.tx->vin)
{
// TODO handle loaded coin being spent notification (GUI)
// Skip notification if spending a loaded coin (will segfault)
// We check IsSpent every time we look up loaded coins for now.
const auto i = mapWallet.find(txin.prevout.hash);
if (i == mapWallet.end())
continue;

CWalletTx &coin = mapWallet[txin.prevout.hash];
coin.BindWallet(this);
NotifyTransactionChanged(this, coin.GetHash(), CT_UPDATED);
Expand Down Expand Up @@ -3303,6 +3332,19 @@ bool CWallet::AddAccountingEntry(const CAccountingEntry& acentry, CWalletDB *pwa
return true;
}

void CWallet::AddLoadedCoins(const std::vector<LoadedCoin>& vLoadedCoin)
{
vLoadedCoinCache.clear();
for (const LoadedCoin& c : vLoadedCoin) {
vLoadedCoinCache.push_back(c);
}
}

std::vector<LoadedCoin> CWallet::GetMyLoadedCoins() const
{
return vLoadedCoinCache;
}

DBErrors CWallet::LoadWallet(bool& fFirstRunRet)
{
LOCK2(cs_main, cs_wallet);
Expand Down
15 changes: 15 additions & 0 deletions src/wallet/wallet.h
Expand Up @@ -79,6 +79,7 @@ class CScheduler;
class CTxMemPool;
class CBlockPolicyEstimator;
class CWalletTx;
class LoadedCoin;
struct FeeCalculation;
enum class FeeEstimateMode;

Expand Down Expand Up @@ -503,6 +504,12 @@ class CInputCoin {
txout = walletTx->tx->vout[i];
}

CInputCoin(const COutPoint& outpointIn, const CTxOut& txoutIn)
{
outpoint = outpointIn;
txout = txoutIn;
}

COutPoint outpoint;
CTxOut txout;

Expand Down Expand Up @@ -750,6 +757,11 @@ class CWallet final : public CCryptoKeyStore, public CValidationInterface
*/
const CBlockIndex* m_last_block_processed;

// TODO improve persistence and caching method of loaded coins tracked by
// wallet.
// Cache of loaded coins owned by this wallet
std::vector<LoadedCoin> vLoadedCoinCache;

public:
/*
* Main wallet lock.
Expand Down Expand Up @@ -996,6 +1008,9 @@ class CWallet final : public CCryptoKeyStore, public CValidationInterface
template <typename ContainerType>
bool DummySignTx(CMutableTransaction &txNew, const ContainerType &coins) const;

void AddLoadedCoins(const std::vector<LoadedCoin>& vLoadedCoin);
std::vector<LoadedCoin> GetMyLoadedCoins() const;

static CFeeRate minTxFee;
static CFeeRate fallbackFee;
static CFeeRate m_discard_rate;
Expand Down
1 change: 1 addition & 0 deletions src/wallet/walletdb.cpp
Expand Up @@ -6,6 +6,7 @@
#include <wallet/walletdb.h>

#include <base58.h>
#include <coins.h>
#include <consensus/tx_verify.h>
#include <consensus/validation.h>
#include <fs.h>
Expand Down
1 change: 1 addition & 0 deletions src/wallet/walletdb.h
Expand Up @@ -7,6 +7,7 @@
#define BITCOIN_WALLET_WALLETDB_H

#include <amount.h>
#include <coins.h>
#include <primitives/transaction.h>
#include <wallet/db.h>
#include <key.h>
Expand Down