[Wallet] Replace -rescan with a new RPC call "rescanblockchain" #7061

Open
wants to merge 2 commits into
from
Jump to file or symbol
Failed to load files and symbols.
+101 −29
Split
@@ -132,7 +132,7 @@
#paytxfee=0.00
# Enable pruning to reduce storage requirements by deleting old blocks.
-# This mode is incompatible with -txindex and -rescan.
+# This mode is incompatible with -txindex.
# 0 = default (no pruning).
# 1 = allows manual pruning via RPC.
# >=550 = target to stay under in MiB.
@@ -21,7 +21,7 @@
REGEX_ARG = re.compile(r'(?:map(?:Multi)?Args(?:\.count\(|\[)|Get(?:Bool)?Arg\()\"(\-[^\"]+?)\"')
REGEX_DOC = re.compile(r'HelpMessageOpt\(\"(\-[^\"=]+?)(?:=|\")')
# list unsupported, deprecated and duplicate args as they need no documentation
-SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize', '-dbcrashratio'])
+SET_DOC_OPTIONAL = set(['-rpcssl', '-benchmark', '-h', '-help', '-socks', '-tor', '-debugnet', '-whitelistalwaysrelay', '-prematurewitness', '-walletprematurewitness', '-promiscuousmempoolflags', '-blockminsize', '-dbcrashratio', '-rescan'])
def main():
used = check_output(CMD_GREP_ARGS, shell=True)
View
@@ -367,7 +367,7 @@ std::string HelpMessage(HelpMessageMode mode)
#ifndef WIN32
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), BITCOIN_PID_FILENAME));
#endif
- strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and -rescan. "
+ strUsage += HelpMessageOpt("-prune=<n>", strprintf(_("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks, and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex and importing keys or addresses may fail."
"Warning: Reverting this setting requires re-downloading the entire blockchain. "
"(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >%u = automatically prune block files to stay under the specified target size in MiB)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
strUsage += HelpMessageOpt("-reindex-chainstate", _("Rebuild chain state from the currently indexed blocks"));
@@ -157,7 +157,7 @@ void TestSendCoins()
wallet.SetAddressBook(test.coinbaseKey.GetPubKey().GetID(), "", "receive");
wallet.AddKeyPubKey(test.coinbaseKey, test.coinbaseKey.GetPubKey());
}
- wallet.ScanForWalletTransactions(chainActive.Genesis(), true);
+ wallet.ScanForWalletTransactions(chainActive.Genesis(), nullptr, true);
wallet.SetBroadcastTransactions(true);
// Create widgets for sending coins and listing transactions.
View
@@ -139,6 +139,8 @@ static const CRPCConvertParam vRPCConvertParams[] =
{ "echojson", 7, "arg7" },
{ "echojson", 8, "arg8" },
{ "echojson", 9, "arg9" },
+ { "rescanblockchain", 0, "startheight"},
+ { "rescanblockchain", 1, "stopheight"},
};
class CRPCConvertTable
View
@@ -165,8 +165,8 @@ bool CDB::Recover(const std::string& filename, void *callbackDataIn, bool (*reco
// Call Salvage with fAggressive=true to
// get as much data as possible.
// Rewrite salvaged data to fresh wallet file
- // Set -rescan so any missing transactions will be
- // found.
+ // call rescanblockchain (RPC) so any missing
@jnewbery

jnewbery May 24, 2017

Member

I can't see where rescanblockchain() is being called. Am I missing something?

@jonasschnelli

jonasschnelli May 25, 2017 edited

Member

It happens outside of this call (somewhere in wallet.cpp). But the comment doesn't say "It" calls rescanblockchain, it either says "you" can call rescanblockchain().

@jnewbery

jnewbery May 25, 2017

Member

I don't understand. The comment is:

    // Recovery procedure:
    // move wallet file to wallet.timestamp.bak
    // Call Salvage with fAggressive=true to
    // get as much data as possible.
    // Rewrite salvaged data to fresh wallet file
    // call rescanblockchain (RPC) so any missing
    // transactions will be found.

This function does:

  • move wallet file to wallet.timestamp.bak
  • Call Salvage with fAggressive=true to get as much data as possible.
  • Rewrite salvaged data to fresh wallet file

but it doesn't call rescanblockchain, and I can't see rescanblockchain being called by the callers.

@jonasschnelli

jonasschnelli May 25, 2017

Member

Yes. You right. The rescan is not part of the call. But the part that calls CDB::Recover does always call a rescan. But now I got your point. We should refer to ScanForWalletTransactions() instead to rescanblockchain (RPC). Right?

@jnewbery

jnewbery May 25, 2017

Member

sorry, I'm still not seeing it. You say "the part that calls CDB::Recover does always call a rescan". Where?

@jonasschnelli

jonasschnelli May 25, 2017 edited

Member

It's confusing. But CDB::Recover only gets calls by setting -salvagewallet which does set -rescan. Maybe I should remove that comment part... I don't know what's best because -rescan is currently mentioned there.

@jnewbery

jnewbery May 25, 2017

Member

ah, makes sense now. Thank you. I wasn't looking in init.cpp, but now I think I see how this fits together. CWallet::Verify() is called first, which calls CWalletDB::Recover() if -salvagewallet is set. Later in AppInitMain(), we call CWallet::AppInitMain(), which is where the rescan happens if -salvagewallet is set.

Perhaps change the comment to something like:

    // Try to recover the wallet:
    // - move wallet file to wallet.timestamp.bak
    // - call Salvage with fAggressive=true to get as much data as possible.
    // - rewrite salvaged data to fresh wallet file
    //
    // CWallet::AppInitMain() will later run a full blockchain rescan to find
    // any missing transactions.
+ // transactions will be found.
int64_t now = GetTime();
newFilename = strprintf("%s.%d.bak", filename, now);
View
@@ -1160,3 +1160,58 @@ UniValue importmulti(const JSONRPCRequest& mainRequest)
return response;
}
+
+UniValue rescanblockchain(const JSONRPCRequest& request)
+{
+ CWallet * const pwallet = GetWalletForJSONRPCRequest(request);
+ if (!EnsureWalletIsAvailable(pwallet, request.fHelp)) {
+ return NullUniValue;
+ }
+
+ if (request.fHelp || request.params.size() > 2) {
+ throw std::runtime_error(
+ "rescanblockchain (\"startheight\") (\"stopheight\")\n"
+ "\nRescan the local blockchain for wallet related transactions.\n"
+ "\nArguments:\n"
+ "1. \"startHeight\" (number, optional) blockheight where the rescan should start\n"
@ryanofsky

ryanofsky Jul 25, 2017

Contributor

Capitalization of height doesn't match actual param name.

+ "2. \"stopHeight\" (number, optional) blockheight where the rescan should stop\n"
@ryanofsky

ryanofsky Jul 25, 2017

Contributor

Could do what pruneblockchain does and allow height to be specified either as a physical height or a time: https://github.com/bitcoin/bitcoin/blob/1caafa6cde3b88d926611771f9b4c06fcc6e0007/src/rpc/blockchain.cpp#L855

+ "\nExamples:\n"
+ + HelpExampleCli("rescanblockchain", "\"100000 120000\"")
+ + HelpExampleRpc("rescanblockchain", "\"100000 120000\"")
+ );
+ }
+
+ LOCK2(cs_main, pwallet->cs_wallet);
+
+ CBlockIndex *pIndexStart = NULL;
+ CBlockIndex *pIndexStop = NULL;
+ if (request.params.size() > 0 && request.params[0].isNum()) {
@ryanofsky

ryanofsky Jul 25, 2017

Contributor

Size checks can be dropped here and below. params[0] will return a null value if the param is missing.

+ pIndexStart = chainActive[request.params[0].get_int()];
+ }
+
+ if (request.params.size() > 1 && request.params[1].isNum()) {
+ pIndexStop = chainActive[request.params[1].get_int()];
+ }
+
+ if (!pIndexStart) {
+ pIndexStart = chainActive.Genesis();
+ }
+
+ //We can't rescan beyond non-pruned blocks, stop and throw an error
+ if (fPruneMode) {
+ CBlockIndex *block = chainActive.Tip();
+ while (block && block->pprev && (block->pprev->nStatus & BLOCK_HAVE_DATA)) {
@ryanofsky

ryanofsky Jul 25, 2017

Contributor

This should check against pIndexStop too, so error won't trigger unnecessarily in cases where missing blocks are noncontinuous (which can happen with manual pruning).

+ block = block->pprev;
+ }
+
+ if (pIndexStart->nHeight < block->nHeight) {
+ throw JSONRPCError(RPC_WALLET_ERROR, "Can't rescan beyond pruned data. Use RPC call getblockchaininfo to determine your pruned height.");
+ }
+ }
+
+ if (pwallet) {
+ pwallet->ScanForWalletTransactions(pIndexStart, pIndexStop, true);
+ }
+
+ return NullUniValue;
+}
View
@@ -3044,6 +3044,7 @@ extern UniValue importwallet(const JSONRPCRequest& request);
extern UniValue importprunedfunds(const JSONRPCRequest& request);
extern UniValue removeprunedfunds(const JSONRPCRequest& request);
extern UniValue importmulti(const JSONRPCRequest& request);
+extern UniValue rescanblockchain(const JSONRPCRequest& request);
static const CRPCCommand commands[] =
{ // category name actor (function) okSafeMode
@@ -3097,6 +3098,7 @@ static const CRPCCommand commands[] =
{ "wallet", "walletpassphrasechange", &walletpassphrasechange, true, {"oldpassphrase","newpassphrase"} },
{ "wallet", "walletpassphrase", &walletpassphrase, true, {"passphrase","timeout"} },
{ "wallet", "removeprunedfunds", &removeprunedfunds, true, {"txid"} },
+ { "wallet", "rescanblockchain", &rescanblockchain, true, {"startheight", "stopheight"} },
{ "generating", "generate", &generate, true, {"nblocks","maxtries"} },
};
View
@@ -1514,7 +1514,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update)
LogPrintf("%s: Rescanning last %i blocks\n", __func__, startBlock ? chainActive.Height() - startBlock->nHeight + 1 : 0);
if (startBlock) {
- const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, update);
+ const CBlockIndex* const failedBlock = ScanForWalletTransactions(startBlock, nullptr, update);
if (failedBlock) {
return failedBlock->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1;
}
@@ -1530,20 +1530,27 @@ int64_t CWallet::RescanFromTime(int64_t startTime, bool update)
* Returns null if scan was successful. Otherwise, if a complete rescan was not
* possible (due to pruning or corruption), returns pointer to the most recent
* block that could not be scanned.
+ *
+ * If pindexStop is not a nullptr, the scan will stop at the block-index
+ * defined by pindexStop
*/
-CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate)
+CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop, bool fUpdate)
{
int64_t nNow = GetTime();
const CChainParams& chainParams = Params();
+ if (pindexStop && pindexStop->nHeight < pindexStart->nHeight) {
+ return nullptr;
+ }
+
CBlockIndex* pindex = pindexStart;
CBlockIndex* ret = nullptr;
{
LOCK2(cs_main, cs_wallet);
fAbortRescan = false;
fScanningWallet = true;
- ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen, if -rescan on startup
+ ShowProgress(_("Rescanning..."), 0); // show rescan progress in GUI as dialog or on splashscreen
double dProgressStart = GuessVerificationProgress(chainParams.TxData(), pindex);
double dProgressTip = GuessVerificationProgress(chainParams.TxData(), chainActive.Tip());
while (pindex && !fAbortRescan)
@@ -1563,6 +1570,9 @@ CBlockIndex* CWallet::ScanForWalletTransactions(CBlockIndex* pindexStart, bool f
} else {
ret = pindex;
}
+ if (pindex == pindexStop) {
+ break;
+ }
pindex = chainActive.Next(pindex);
}
if (pindex && fAbortRescan) {
@@ -3831,7 +3841,6 @@ std::string CWallet::GetWalletHelpString(bool showDebug)
CURRENCY_UNIT, FormatMoney(DEFAULT_TRANSACTION_MINFEE)));
strUsage += HelpMessageOpt("-paytxfee=<amt>", strprintf(_("Fee (in %s/kB) to add to transactions you send (default: %s)"),
CURRENCY_UNIT, FormatMoney(payTxFee.GetFeePerK())));
- strUsage += HelpMessageOpt("-rescan", _("Rescan the block chain for missing wallet transactions on startup"));
strUsage += HelpMessageOpt("-salvagewallet", _("Attempt to recover private keys from a corrupt wallet on startup"));
strUsage += HelpMessageOpt("-spendzeroconfchange", strprintf(_("Spend unconfirmed change when sending transactions (default: %u)"), DEFAULT_SPEND_ZEROCONF_CHANGE));
strUsage += HelpMessageOpt("-txconfirmtarget=<n>", strprintf(_("If paytxfee is not set, include enough fee so transactions begin confirmation on average within n blocks (default: %u)"), DEFAULT_TX_CONFIRM_TARGET));
@@ -3841,7 +3850,7 @@ std::string CWallet::GetWalletHelpString(bool showDebug)
strUsage += HelpMessageOpt("-wallet=<file>", _("Specify wallet file (within data directory)") + " " + strprintf(_("(default: %s)"), DEFAULT_WALLET_DAT));
strUsage += HelpMessageOpt("-walletbroadcast", _("Make the wallet broadcast transactions") + " " + strprintf(_("(default: %u)"), DEFAULT_WALLETBROADCAST));
strUsage += HelpMessageOpt("-walletnotify=<cmd>", _("Execute command when a wallet transaction changes (%s in cmd is replaced by TxID)"));
- strUsage += HelpMessageOpt("-zapwallettxes=<mode>", _("Delete all wallet transactions and only recover those parts of the blockchain through -rescan on startup") +
+ strUsage += HelpMessageOpt("-zapwallettxes=<mode>", _("Delete all wallet transactions and only recover those parts of the blockchain through a rescan on startup") +
" " + _("(1 = keep tx meta data e.g. account owner and payment request information, 2 = drop tx meta data)"));
if (showDebug)
@@ -3884,12 +3893,19 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile)
std::unique_ptr<CWalletDBWrapper> dbw(new CWalletDBWrapper(&bitdb, walletFile));
CWallet *walletInstance = new CWallet(std::move(dbw));
DBErrors nLoadWalletRet = walletInstance->LoadWallet(fFirstRun);
+ bool needsRescan = false;
if (nLoadWalletRet != DB_LOAD_OK)
{
if (nLoadWalletRet == DB_CORRUPT) {
InitError(strprintf(_("Error loading %s: Wallet corrupted"), walletFile));
return NULL;
}
+ else if (nLoadWalletRet == DB_NEED_RESCAN)
+ {
+ InitWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data might be missing or incorrect. Rescanning..."),
+ walletFile));
+ needsRescan = true;
+ }
else if (nLoadWalletRet == DB_NONCRITICAL_ERROR)
{
InitWarning(strprintf(_("Error reading %s! All keys read correctly, but transaction data"
@@ -3971,7 +3987,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile)
RegisterValidationInterface(walletInstance);
CBlockIndex *pindexRescan = chainActive.Genesis();
- if (!GetBoolArg("-rescan", false))
@ryanofsky

ryanofsky May 24, 2017

Contributor

Is there any advantage to getting rid of the -rescan option if we still have keep all the rescan logic below? Is there a future cleanup or new feature that will be easier to implement with the option gone?

@jonasschnelli

jonasschnelli May 24, 2017

Member

The -rescan option is global while this PR changes it (partially) to per-wallet. Still, -zapwallettx, etc. need to also be moved to per-wallet basis (I don't know how right now).

But removing the global -rescan option has to be done sooner or later if we want proper multiwallet.

@ryanofsky

ryanofsky May 24, 2017

Contributor

But removing the global -rescan option has to be done sooner or later if we want proper multiwallet.

Do -zapwallettx, etc also need to be removed to support proper multiwallet?

@jonasschnelli

jonasschnelli May 24, 2017

Member

IMO Yes. It's pure db utility. Either we move it to RPC or to a new wallet/db tool.

@jnewbery

jnewbery May 24, 2017

Member

This is confusing: the if statement requires !needsRescan. Shouldn't that be the other way around? If needsRescan is true, we should rescan. I think the if statement should be:

if (needsRescan || GetBoolArg("-salvagewallet", true) || GetBoolArg("-zapwallettxes", true))

re: zapwallet and salvagewallet. I agree that they should be changed to RPCs for multiwallet.

@jonasschnelli

jonasschnelli May 25, 2017

Member

No. I don't think so.
If that if statement is true, we load the wallet best block which prevents a complete rescan (so it's the opposite).
So we only load the wallet best block (== no rescan) if needsRescan is false and -salvagewallet and --zapwallettxes` have not been set.

@jnewbery

jnewbery May 25, 2017

Member

Yes, you're right.

+ if (!needsRescan && !GetBoolArg("-salvagewallet", false) && !GetBoolArg("-zapwallettxes", false))
{
CWalletDB walletdb(*walletInstance->dbw);
CBlockLocator locator;
@@ -4005,7 +4021,7 @@ CWallet* CWallet::CreateWalletFromFile(const std::string walletFile)
}
nStart = GetTimeMillis();
- walletInstance->ScanForWalletTransactions(pindexRescan, true);
+ walletInstance->ScanForWalletTransactions(pindexRescan, nullptr, true);
LogPrintf(" rescan %15dms\n", GetTimeMillis() - nStart);
walletInstance->SetBestChain(chainActive.GetLocator());
walletInstance->dbw->IncrementUpdateCounter();
@@ -4084,6 +4100,10 @@ bool CWallet::ParameterInteraction()
SoftSetArg("-wallet", DEFAULT_WALLET_DAT);
const bool is_multiwallet = gArgs.GetArgs("-wallet").size() > 1;
+ if (GetBoolArg("-rescan", false)) {
+ return InitError(_("-rescan as startup argument is no longer supported. Use rescanblockchain RPC call"));
+ }
+
if (GetBoolArg("-disablewallet", DEFAULT_DISABLE_WALLET))
return true;
@@ -4095,10 +4115,6 @@ bool CWallet::ParameterInteraction()
if (is_multiwallet) {
return InitError(strprintf("%s is only allowed with a single wallet file", "-salvagewallet"));
}
- // Rewrite just private keys: rescan to find transactions
- if (SoftSetBoolArg("-rescan", true)) {
- LogPrintf("%s: parameter interaction: -salvagewallet=1 -> setting -rescan=1\n", __func__);
- }
}
int zapwallettxes = GetArg("-zapwallettxes", 0);
@@ -4107,14 +4123,10 @@ bool CWallet::ParameterInteraction()
LogPrintf("%s: parameter interaction: -zapwallettxes=%s -> setting -persistmempool=0\n", __func__, zapwallettxes);
}
- // -zapwallettxes implies a rescan
if (zapwallettxes != 0) {
if (is_multiwallet) {
return InitError(strprintf("%s is only allowed with a single wallet file", "-zapwallettxes"));
}
- if (SoftSetBoolArg("-rescan", true)) {
- LogPrintf("%s: parameter interaction: -zapwallettxes=%s -> setting -rescan=1\n", __func__, zapwallettxes);
- }
}
if (is_multiwallet) {
@@ -4125,8 +4137,6 @@ bool CWallet::ParameterInteraction()
if (GetBoolArg("-sysperms", false))
return InitError("-sysperms is not allowed in combination with enabled wallet functionality");
- if (GetArg("-prune", 0) && GetBoolArg("-rescan", false))
- return InitError(_("Rescans are not possible in pruned mode. You will need to use -reindex which will download the whole blockchain again."));
if (::minRelayTxFee.GetFeePerK() > HIGH_TX_FEE_PER_KB)
InitWarning(AmountHighWarn("-minrelaytxfee") + " " +
View
@@ -934,7 +934,7 @@ class CWallet : public CCryptoKeyStore, public CValidationInterface
void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock) override;
bool AddToWalletIfInvolvingMe(const CTransactionRef& tx, const CBlockIndex* pIndex, int posInBlock, bool fUpdate);
int64_t RescanFromTime(int64_t startTime, bool update);
- CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, bool fUpdate = false);
+ CBlockIndex* ScanForWalletTransactions(CBlockIndex* pindexStart, CBlockIndex* pindexStop = nullptr, bool fUpdate = false);
void ReacceptWalletTransactions();
void ResendWalletTransactions(int64_t nBestBlockTime, CConnman* connman) override;
std::vector<uint256> ResendWalletTransactionsBefore(int64_t nTime, CConnman* connman);
View
@@ -571,9 +571,10 @@ DBErrors CWalletDB::LoadWallet(CWallet* pwallet)
{
// Leave other errors alone, if we try to fix them we might make things worse.
fNoncriticalErrors = true; // ... but do warn the user there is something wrong.
- if (strType == "tx")
+ if (strType == "tx") {
// Rescan if there is a bad transaction record:
- SoftSetBoolArg("-rescan", true);
+ result = DB_NEED_RESCAN;
+ }
}
}
if (!strErr.empty())
View
@@ -53,7 +53,8 @@ enum DBErrors
DB_NONCRITICAL_ERROR,
DB_TOO_NEW,
DB_LOAD_FAIL,
- DB_NEED_REWRITE
+ DB_NEED_REWRITE,
+ DB_NEED_RESCAN,
};
/* simple HD chain data model */
@@ -88,7 +88,9 @@ def run_test (self):
# Needs rescan
self.stop_node(1)
- self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1] + ['-rescan'])
+ self.nodes[1] = self.start_node(1, self.options.tmpdir, self.extra_args[1])
+ self.nodes[1].rescanblockchain()
+
#connect_nodes_bi(self.nodes, 0, 1)
assert_equal(self.nodes[1].getbalance(), num_hd_adds + 1)
@@ -338,7 +338,6 @@ def run_test(self):
# maintenance tests
maintenance = [
- '-rescan',
'-reindex',
'-zapwallettxes=1',
'-zapwallettxes=2',