Skip to content

Commit

Permalink
Merge bitcoin#9951: Wallet database handling abstractions/simplificat…
Browse files Browse the repository at this point in the history 
…ions

911a480 wallet: Add comment describing the various classes in walletdb.h (Wladimir J. van der Laan)
69d2e9b wallet: Make IsDummy private in CWalletDBWrapper (Wladimir J. van der Laan)
3323281 wallet: CWalletDB CDB composition not inheritance (Wladimir J. van der Laan)
be9e1a9 wallet: Reduce references to global bitdb environment (Wladimir J. van der Laan)
071c955 wallet: Get rid of fFileBacked (Wladimir J. van der Laan)
71afe3c wallet: Introduce database handle wrapper (Wladimir J. van der Laan)

Tree-SHA512: e4e72953c61a2f6995d609a32f8ed8e18cab9a92bc9e193d46a1d1f06d9daa5c6da6fce2867d4e3ba4fc0439141901a3d35f246486f0fa8f59587786379dfcbd
  • Loading branch information
@laanwj @PastaPastaPasta
laanwj authored and PastaPastaPasta committed May 23, 2019
1 parent f7e6022 commit 104e6fa
Show file tree
Hide file tree
Showing 9 changed files with 350 additions and 236 deletions.
3 changes: 2 additions & 1 deletion src/qt/test/wallettests.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,8 @@ void WalletTests::walletTests()
TestChain100Setup test;
test.CreateAndProcessBlock({}, GetScriptForRawPubKey(test.coinbaseKey.GetPubKey()));
bitdb.MakeMock();
CWallet wallet("wallet_test.dat");
std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat"));
CWallet wallet(std::move(dbw));
bool firstRun;
wallet.LoadWallet(firstRun);
{
Expand Down
126 changes: 94 additions & 32 deletions src/wallet/db.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,31 +358,34 @@ void CDBEnv::CheckpointLSN(const std::string& strFile)
}


CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL)
CDB::CDB(CWalletDBWrapper& dbw, const char* pszMode, bool fFlushOnCloseIn) : pdb(NULL), activeTxn(NULL)
{
int ret;
fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w'));
fFlushOnClose = fFlushOnCloseIn;
if (strFilename.empty())
env = dbw.env;
if (dbw.IsDummy()) {
return;
}
const std::string &strFilename = dbw.strFile;

bool fCreate = strchr(pszMode, 'c') != NULL;
unsigned int nFlags = DB_THREAD;
if (fCreate)
nFlags |= DB_CREATE;

{
LOCK(bitdb.cs_db);
if (!bitdb.Open(GetDataDir()))
LOCK(env->cs_db);
if (!env->Open(GetDataDir()))
throw std::runtime_error("CDB: Failed to open database environment.");

strFile = strFilename;
++bitdb.mapFileUseCount[strFile];
pdb = bitdb.mapDb[strFile];
++env->mapFileUseCount[strFile];
pdb = env->mapDb[strFile];
if (pdb == NULL) {
pdb = new Db(bitdb.dbenv, 0);
pdb = new Db(env->dbenv, 0);

bool fMockDb = bitdb.IsMock();
bool fMockDb = env->IsMock();
if (fMockDb) {
DbMpoolFile* mpf = pdb->get_mpf();
ret = mpf->set_flags(DB_MPOOL_NOFILE, 1);
Expand All @@ -400,7 +403,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose
if (ret != 0) {
delete pdb;
pdb = NULL;
--bitdb.mapFileUseCount[strFile];
--env->mapFileUseCount[strFile];
strFile = "";
throw std::runtime_error(strprintf("CDB: Error %d, can't open database %s", ret, strFilename));
}
Expand All @@ -412,7 +415,7 @@ CDB::CDB(const std::string& strFilename, const char* pszMode, bool fFlushOnClose
fReadOnly = fTmp;
}

bitdb.mapDb[strFile] = pdb;
env->mapDb[strFile] = pdb;
}
}
}
Expand All @@ -427,7 +430,7 @@ void CDB::Flush()
if (fReadOnly)
nMinutes = 1;

bitdb.dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
env->dbenv->txn_checkpoint(nMinutes ? GetArg("-dblogsize", DEFAULT_WALLET_DBLOGSIZE) * 1024 : 0, nMinutes, 0);
}

void CDB::Close()
Expand All @@ -443,8 +446,8 @@ void CDB::Close()
Flush();

{
LOCK(bitdb.cs_db);
--bitdb.mapFileUseCount[strFile];
LOCK(env->cs_db);
--env->mapFileUseCount[strFile];
}
}

Expand All @@ -471,23 +474,28 @@ bool CDBEnv::RemoveDb(const std::string& strFile)
return (rc == 0);
}

bool CDB::Rewrite(const std::string& strFile, const char* pszSkip)
bool CDB::Rewrite(CWalletDBWrapper& dbw, const char* pszSkip)
{
if (dbw.IsDummy()) {
return true;
}
CDBEnv *env = dbw.env;
const std::string& strFile = dbw.strFile;
while (true) {
{
LOCK(bitdb.cs_db);
if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) {
LOCK(env->cs_db);
if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0) {
// Flush log data to the dat file
bitdb.CloseDb(strFile);
bitdb.CheckpointLSN(strFile);
bitdb.mapFileUseCount.erase(strFile);
env->CloseDb(strFile);
env->CheckpointLSN(strFile);
env->mapFileUseCount.erase(strFile);

bool fSuccess = true;
LogPrintf("CDB::Rewrite: Rewriting %s...\n", strFile);
std::string strFileRes = strFile + ".rewrite";
{ // surround usage of db with extra {}
CDB db(strFile.c_str(), "r");
Db* pdbCopy = new Db(bitdb.dbenv, 0);
CDB db(dbw, "r");
Db* pdbCopy = new Db(env->dbenv, 0);

int ret = pdbCopy->open(NULL, // Txn pointer
strFileRes.c_str(), // Filename
Expand Down Expand Up @@ -530,17 +538,17 @@ bool CDB::Rewrite(const std::string& strFile, const char* pszSkip)
}
if (fSuccess) {
db.Close();
bitdb.CloseDb(strFile);
env->CloseDb(strFile);
if (pdbCopy->close(0))
fSuccess = false;
delete pdbCopy;
}
}
if (fSuccess) {
Db dbA(bitdb.dbenv, 0);
Db dbA(env->dbenv, 0);
if (dbA.remove(strFile.c_str(), NULL, 0))
fSuccess = false;
Db dbB(bitdb.dbenv, 0);
Db dbB(env->dbenv, 0);
if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0))
fSuccess = false;
}
Expand Down Expand Up @@ -595,16 +603,21 @@ void CDBEnv::Flush(bool fShutdown)
}
}

bool CDB::PeriodicFlush(std::string strFile)
bool CDB::PeriodicFlush(CWalletDBWrapper& dbw)
{
if (dbw.IsDummy()) {
return true;
}
bool ret = false;
CDBEnv *env = dbw.env;
const std::string& strFile = dbw.strFile;
TRY_LOCK(bitdb.cs_db,lockDb);
if (lockDb)
{
// Don't do this if any databases are in use
int nRefCount = 0;
std::map<std::string, int>::iterator mit = bitdb.mapFileUseCount.begin();
while (mit != bitdb.mapFileUseCount.end())
std::map<std::string, int>::iterator mit = env->mapFileUseCount.begin();
while (mit != env->mapFileUseCount.end())
{
nRefCount += (*mit).second;
mit++;
Expand All @@ -613,17 +626,17 @@ bool CDB::PeriodicFlush(std::string strFile)
if (nRefCount == 0)
{
boost::this_thread::interruption_point();
std::map<std::string, int>::iterator mi = bitdb.mapFileUseCount.find(strFile);
if (mi != bitdb.mapFileUseCount.end())
std::map<std::string, int>::iterator mi = env->mapFileUseCount.find(strFile);
if (mi != env->mapFileUseCount.end())
{
LogPrint(BCLog::DB, "Flushing %s\n", strFile);
int64_t nStart = GetTimeMillis();

// Flush wallet file so it's self contained
bitdb.CloseDb(strFile);
bitdb.CheckpointLSN(strFile);
env->CloseDb(strFile);
env->CheckpointLSN(strFile);

bitdb.mapFileUseCount.erase(mi++);
env->mapFileUseCount.erase(mi++);
LogPrint(BCLog::DB, "Flushed %s %dms\n", strFile, GetTimeMillis() - nStart);
ret = true;
}
Expand All @@ -632,3 +645,52 @@ bool CDB::PeriodicFlush(std::string strFile)

return ret;
}

bool CWalletDBWrapper::Rewrite(const char* pszSkip)
{
return CDB::Rewrite(*this, pszSkip);
}

bool CWalletDBWrapper::Backup(const std::string& strDest)
{
if (IsDummy()) {
return false;
}
while (true)
{
{
LOCK(env->cs_db);
if (!env->mapFileUseCount.count(strFile) || env->mapFileUseCount[strFile] == 0)
{
// Flush log data to the dat file
env->CloseDb(strFile);
env->CheckpointLSN(strFile);
env->mapFileUseCount.erase(strFile);

// Copy wallet file
fs::path pathSrc = GetDataDir() / strFile;
fs::path pathDest(strDest);
if (fs::is_directory(pathDest))
pathDest /= strFile;

try {
fs::copy_file(pathSrc, pathDest, fs::copy_option::overwrite_if_exists);
LogPrintf("copied %s to %s\n", strFile, pathDest.string());
return true;
} catch (const fs::filesystem_error& e) {
LogPrintf("error copying %s to %s - %s\n", strFile, pathDest.string(), e.what());
return false;
}
}
}
MilliSleep(100);
}
return false;
}

void CWalletDBWrapper::Flush(bool shutdown)
{
if (!IsDummy()) {
env->Flush(shutdown);
}
}
59 changes: 53 additions & 6 deletions src/wallet/db.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,52 @@ class CDBEnv

extern CDBEnv bitdb;

/** An instance of this class represents one database.
* For BerkeleyDB this is just a (env, strFile) tuple.
**/
class CWalletDBWrapper
{
friend class CDB;
public:
/** Create dummy DB handle */
CWalletDBWrapper(): env(nullptr)
{
}

/** Create DB handle to real database */
CWalletDBWrapper(CDBEnv *env_in, const std::string &strFile_in):
env(env_in), strFile(strFile_in)
{
}

/** Rewrite the entire database on disk, with the exception of key pszSkip if non-zero
*/
bool Rewrite(const char* pszSkip=nullptr);

/** Back up the entire database to a file.
*/
bool Backup(const std::string& strDest);

/** Get a name for this database, for debugging etc.
*/
std::string GetName() const { return strFile; }

/** Make sure all changes are flushed to disk.
*/
void Flush(bool shutdown);

private:
/** BerkeleyDB specific */
CDBEnv *env;
std::string strFile;

/** Return whether this database handle is a dummy for testing.
* Only to be used at a low level, application should ideally not care
* about this.
*/
bool IsDummy() { return env == nullptr; }
};


/** RAII class that provides access to a Berkeley database */
class CDB
Expand All @@ -97,18 +143,19 @@ class CDB
DbTxn* activeTxn;
bool fReadOnly;
bool fFlushOnClose;
CDBEnv *env;

explicit CDB(const std::string& strFilename, const char* pszMode = "r+", bool fFlushOnCloseIn=true);
public:
explicit CDB(CWalletDBWrapper& dbw, const char* pszMode = "r+", bool fFlushOnCloseIn=true);
~CDB() { Close(); }

public:
void Flush();
void Close();
static bool Recover(const std::string& filename, void *callbackDataIn, bool (*recoverKVcallback)(void* callbackData, CDataStream ssKey, CDataStream ssValue));

/* flush the wallet passively (TRY_LOCK)
ideal to be called periodically */
static bool PeriodicFlush(std::string strFile);
static bool PeriodicFlush(CWalletDBWrapper& dbw);
/* verifies the database environment */
static bool VerifyEnvironment(const std::string& walletFile, const boost::filesystem::path& dataDir, std::string& errorStr);
/* verifies the database file */
Expand All @@ -118,7 +165,7 @@ class CDB
CDB(const CDB&);
void operator=(const CDB&);

protected:
public:
template <typename K, typename T>
bool Read(const K& key, T& value)
{
Expand Down Expand Up @@ -158,7 +205,7 @@ class CDB
bool Write(const K& key, const T& value, bool fOverwrite = true)
{
if (!pdb)
return false;
return true;
if (fReadOnly)
assert(!"Write called on database in read-only mode");

Expand Down Expand Up @@ -312,7 +359,7 @@ class CDB
return Write(std::string("version"), nVersion);
}

bool static Rewrite(const std::string& strFile, const char* pszSkip = NULL);
bool static Rewrite(CWalletDBWrapper& dbw, const char* pszSkip = NULL);
};

#endif // BITCOIN_WALLET_DB_H
2 changes: 1 addition & 1 deletion src/wallet/rpcwallet.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2133,7 +2133,7 @@ UniValue walletpassphrase(const JSONRPCRequest& request)
pwallet->TopUpKeyPool();

pwallet->nRelockTime = GetTime() + nSleepTime;
RPCRunLater(strprintf("lockwallet(%s)", pwallet->strWalletFile), boost::bind(LockWallet, pwallet), nSleepTime);
RPCRunLater(strprintf("lockwallet(%s)", pwallet->GetName()), boost::bind(LockWallet, pwallet), nSleepTime);

return NullUniValue;
}
Expand Down
3 changes: 2 additions & 1 deletion src/wallet/test/wallet_test_fixture.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ WalletTestingSetup::WalletTestingSetup(const std::string& chainName):
bitdb.MakeMock();

bool fFirstRun;
pwalletMain = new CWallet("wallet_test.dat");
std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, "wallet_test.dat"));
pwalletMain = new CWallet(std::move(dbw));
pwalletMain->LoadWallet(fFirstRun);
RegisterValidationInterface(pwalletMain);

Expand Down
Loading

0 comments on commit 104e6fa

Please sign in to comment.