Permalink
Browse files

Merge #7917: Optimize reindex

b4d24e1 Report reindexing progress in GUI (Pieter Wuille)
d3d7547 Add -reindex-chainstate that does not rebuild block index (Pieter Wuille)
fb8fad1 Optimize ActivateBestChain for long chains (Pieter Wuille)
316623f Switch reindexing to AcceptBlock in-loop and ActivateBestChain afterwards (Pieter Wuille)
d253ec4 Make ProcessNewBlock dbp const and update comment (Pieter Wuille)
  • Loading branch information...
laanwj committed May 18, 2016
2 parents 8e8bebc + b4d24e1 commit 239d4198645434542b01916dae5eebc7a3a7bbcc
View
@@ -4,10 +4,11 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
#
# Test -reindex with CheckBlockIndex
# Test -reindex and -reindex-chainstate with CheckBlockIndex
#
from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *
import time
class ReindexTest(BitcoinTestFramework):
@@ -20,13 +21,22 @@ def setup_network(self):
self.is_network_split = False
self.nodes.append(start_node(0, self.options.tmpdir))
def run_test(self):
def reindex(self, justchainstate=False):
self.nodes[0].generate(3)
blockcount = self.nodes[0].getblockcount()
stop_node(self.nodes[0], 0)
wait_bitcoinds()
self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex", "-checkblockindex=1"])
assert_equal(self.nodes[0].getblockcount(), 3)
self.nodes[0]=start_node(0, self.options.tmpdir, ["-debug", "-reindex-chainstate" if justchainstate else "-reindex", "-checkblockindex=1"])
while self.nodes[0].getblockcount() < blockcount:
time.sleep(0.1)
assert_equal(self.nodes[0].getblockcount(), blockcount)
print("Success")
def run_test(self):
self.reindex(False)
self.reindex(True)
self.reindex(False)
self.reindex(True)
if __name__ == '__main__':
ReindexTest().main()
View
@@ -327,7 +327,8 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by pruning (deleting) old blocks. This mode is incompatible with -txindex and -rescan. "
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
"(default: 0 = disable pruning blocks, >%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files on startup"));
strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks"));
strUsage += HelpMessageOpt("-reindex", _("Rebuild chain state and block index from the blk*.dat files on disk"));
#ifndef WIN32
strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)"));
#endif
@@ -404,7 +405,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
}
string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http, libevent, tor, zmq"; // Don't translate these and qt below
string debugCategories = "addrman, alert, bench, coindb, db, http, libevent, lock, mempool, mempoolrej, net, proxy, prune, rand, reindex, rpc, selectcoins, tor, zmq"; // Don't translate these and qt below
if (mode == HMM_BITCOIN_QT)
debugCategories += ", qt";
strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " +
@@ -554,9 +555,10 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
{
const CChainParams& chainparams = Params();
RenameThread("bitcoin-loadblk");
CImportingNow imp;
// -reindex
if (fReindex) {
CImportingNow imp;
int nFile = 0;
while (true) {
CDiskBlockPos pos(nFile, 0);
@@ -581,7 +583,6 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
if (boost::filesystem::exists(pathBootstrap)) {
FILE *file = fopen(pathBootstrap.string().c_str(), "rb");
if (file) {
CImportingNow imp;
boost::filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old";
LogPrintf("Importing bootstrap.dat...\n");
LoadExternalBlockFile(chainparams, file);
@@ -595,14 +596,20 @@ void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
BOOST_FOREACH(const boost::filesystem::path& path, vImportFiles) {
FILE *file = fopen(path.string().c_str(), "rb");
if (file) {
CImportingNow imp;
LogPrintf("Importing blocks file %s...\n", path.string());
LoadExternalBlockFile(chainparams, file);
} else {
LogPrintf("Warning: Could not open blocks file %s\n", path.string());
}
}
// scan for better chains in the block chain database, that are not yet connected in the active best chain
CValidationState state;
if (!ActivateBestChain(state, chainparams)) {
LogPrintf("Failed to connect best block");
StartShutdown();
}
if (GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) {
LogPrintf("Stopping after block import\n");
StartShutdown();
@@ -1158,6 +1165,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
// ********************************************************* Step 7: load block chain
fReindex = GetBoolArg("-reindex", false);
bool fReindexChainState = GetBoolArg("-reindex-chainstate", false);
// Upgrading to 0.8; hard-link the old blknnnn.dat files into /blocks/
boost::filesystem::path blocksDir = GetDataDir() / "blocks";
@@ -1219,7 +1227,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
delete pblocktree;
pblocktree = new CBlockTreeDB(nBlockTreeDBCache, false, fReindex);
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex);
pcoinsdbview = new CCoinsViewDB(nCoinDBCache, false, fReindex || fReindexChainState);
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
@@ -1248,7 +1256,7 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
// Check for changed -txindex state
if (fTxIndex != GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
strLoadError = _("You need to rebuild the database using -reindex to change -txindex");
strLoadError = _("You need to rebuild the database using -reindex-chainstate to change -txindex");
break;
}
@@ -1358,12 +1366,6 @@ bool AppInit2(boost::thread_group& threadGroup, CScheduler& scheduler)
if (mapArgs.count("-blocknotify"))
uiInterface.NotifyBlockTip.connect(BlockNotifyCallback);
uiInterface.InitMessage(_("Activating best chain..."));
// scan for better chains in the block chain database, that are not yet connected in the active best chain
CValidationState state;
if (!ActivateBestChain(state, chainparams))
strErrors << "Failed to connect best block";
std::vector<boost::filesystem::path> vImportFiles;
if (mapArgs.count("-loadblock"))
{
View
@@ -2811,10 +2811,9 @@ static void PruneBlockIndexCandidates() {
* Try to make some progress towards making pindexMostWork the active block.
* pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork.
*/
static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock)
static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound)
{
AssertLockHeld(cs_main);
bool fInvalidFound = false;
const CBlockIndex *pindexOldTip = chainActive.Tip();
const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork);
@@ -2884,6 +2883,28 @@ static bool ActivateBestChainStep(CValidationState& state, const CChainParams& c
return true;
}
static void NotifyHeaderTip() {
bool fNotify = false;
bool fInitialBlockDownload = false;
static CBlockIndex* pindexHeaderOld = NULL;
CBlockIndex* pindexHeader = NULL;
{
LOCK(cs_main);
if (!setBlockIndexCandidates.empty()) {
pindexHeader = *setBlockIndexCandidates.rbegin();
}
if (pindexHeader != pindexHeaderOld) {
fNotify = true;
fInitialBlockDownload = IsInitialBlockDownload();
pindexHeaderOld = pindexHeader;
}
}
// Send block tip changed notifications without cs_main
if (fNotify) {
uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader);
}
}
/**
* Make the best chain active, in multiple steps. The result is either failure
* or an activated best chain. pblock is either NULL or a pointer to a block
@@ -2902,15 +2923,22 @@ bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams,
{
LOCK(cs_main);
CBlockIndex *pindexOldTip = chainActive.Tip();
pindexMostWork = FindMostWorkChain();
if (pindexMostWork == NULL) {
pindexMostWork = FindMostWorkChain();
}
// Whether we have anything to do at all.
if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip())
return true;
if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL))
bool fInvalidFound = false;
if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, fInvalidFound))
return false;
if (fInvalidFound) {
// Wipe cache, we may need another branch now.
pindexMostWork = NULL;
}
pindexNewTip = chainActive.Tip();
pindexFork = chainActive.FindFork(pindexOldTip);
fInitialDownload = IsInitialBlockDownload();
@@ -3398,11 +3426,12 @@ static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state
}
/** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */
static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, CDiskBlockPos* dbp)
static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp)
{
AssertLockHeld(cs_main);
CBlockIndex *&pindex = *ppindex;
CBlockIndex *pindexDummy = NULL;
CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy;
if (!AcceptBlockHeader(block, state, chainparams, &pindex))
return false;
@@ -3474,7 +3503,7 @@ static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned
}
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp)
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp)
{
{
LOCK(cs_main);
@@ -3492,6 +3521,8 @@ bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, c
return error("%s: AcceptBlock FAILED", __func__);
}
NotifyHeaderTip();
if (!ActivateBestChain(state, chainparams, pblock))
return error("%s: ActivateBestChain failed", __func__);
@@ -4037,15 +4068,26 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
// process in case the block isn't known yet
if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) {
LOCK(cs_main);
CValidationState state;
if (ProcessNewBlock(state, chainparams, NULL, &block, true, dbp))
if (AcceptBlock(block, state, chainparams, NULL, true, dbp))
nLoaded++;
if (state.IsError())
break;
} else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) {
LogPrintf("Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight);
LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight);
}
// Activate the genesis block so normal node progress can continue
if (hash == chainparams.GetConsensus().hashGenesisBlock) {
CValidationState state;
if (!ActivateBestChain(state, chainparams)) {
break;
}
}
NotifyHeaderTip();
// Recursively process earlier encountered successors of this block
deque<uint256> queue;
queue.push_back(hash);
@@ -4057,17 +4099,19 @@ bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskB
std::multimap<uint256, CDiskBlockPos>::iterator it = range.first;
if (ReadBlockFromDisk(block, it->second, chainparams.GetConsensus()))
{
LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(),
LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(),
head.ToString());
LOCK(cs_main);
CValidationState dummy;
if (ProcessNewBlock(dummy, chainparams, NULL, &block, true, &it->second))
if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second))
{
nLoaded++;
queue.push_back(block.GetHash());
}
}
range.first++;
mapBlocksUnknownParent.erase(it);
NotifyHeaderTip();
}
}
} catch (const std::exception& e) {
@@ -5077,6 +5121,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
ReadCompactSize(vRecv); // ignore tx count; assume it is 0.
}
{
LOCK(cs_main);
if (nCount == 0) {
@@ -5167,6 +5212,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
}
CheckBlockIndex(chainparams.GetConsensus());
}
NotifyHeaderTip();
}
else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing
View
@@ -212,10 +212,10 @@ void UnregisterNodeSignals(CNodeSignals& nodeSignals);
* @param[in] pfrom The node which we are receiving the block from; it is added to mapBlockSource and may be penalised if the block is invalid.
* @param[in] pblock The block we want to process.
* @param[in] fForceProcessing Process this block even if unrequested; used for non-network block sources and whitelisted peers.
* @param[out] dbp If pblock is stored to disk (or already there), this will be set to its location.
* @param[out] dbp The already known disk position of pblock, or NULL if not yet stored.
* @return True if state.IsValid()
*/
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, CDiskBlockPos* dbp);
bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, const CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp);
/** Check whether enough disk space is available for an incoming block */
bool CheckDiskSpace(uint64_t nAdditionalBytes = 0);
/** Open a block file (blk?????.dat) */
View
@@ -457,8 +457,8 @@ void BitcoinGUI::setClientModel(ClientModel *clientModel)
setNumConnections(clientModel->getNumConnections());
connect(clientModel, SIGNAL(numConnectionsChanged(int)), this, SLOT(setNumConnections(int)));
setNumBlocks(clientModel->getNumBlocks(), clientModel->getLastBlockDate(), clientModel->getVerificationProgress(NULL));
connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double)), this, SLOT(setNumBlocks(int,QDateTime,double)));
setNumBlocks(clientModel->getNumBlocks(), clientModel->getLastBlockDate(), clientModel->getVerificationProgress(NULL), false);
connect(clientModel, SIGNAL(numBlocksChanged(int,QDateTime,double,bool)), this, SLOT(setNumBlocks(int,QDateTime,double,bool)));
// Receive and report messages from client model
connect(clientModel, SIGNAL(message(QString,QString,unsigned int)), this, SLOT(message(QString,QString,unsigned int)));
@@ -696,7 +696,7 @@ void BitcoinGUI::setNumConnections(int count)
labelConnectionsIcon->setToolTip(tr("%n active connection(s) to Bitcoin network", "", count));
}
void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress)
void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool header)
{
if(!clientModel)
return;
@@ -708,15 +708,25 @@ void BitcoinGUI::setNumBlocks(int count, const QDateTime& blockDate, double nVer
enum BlockSource blockSource = clientModel->getBlockSource();
switch (blockSource) {
case BLOCK_SOURCE_NETWORK:
if (header) {
return;
}
progressBarLabel->setText(tr("Synchronizing with network..."));
break;
case BLOCK_SOURCE_DISK:
progressBarLabel->setText(tr("Importing blocks from disk..."));
if (header) {
progressBarLabel->setText(tr("Indexing blocks on disk..."));
} else {
progressBarLabel->setText(tr("Processing blocks on disk..."));
}
break;
case BLOCK_SOURCE_REINDEX:
progressBarLabel->setText(tr("Reindexing blocks on disk..."));
break;
case BLOCK_SOURCE_NONE:
if (header) {
return;
}
// Case: not Importing, not Reindexing and no network connection
progressBarLabel->setText(tr("No block source available..."));
break;
View
@@ -150,7 +150,7 @@ public Q_SLOTS:
/** Set number of connections shown in the UI */
void setNumConnections(int count);
/** Set number of blocks and last block date shown in the UI */
void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress);
void setNumBlocks(int count, const QDateTime& blockDate, double nVerificationProgress, bool headers);
/** Notify the user of an event from the core network or transaction handling code.
@param[in] title the message box / notification title
Oops, something went wrong.

0 comments on commit 239d419

Please sign in to comment.