From 6ea23b2bb8e14c50d1a852417b389e992817a05e Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 12 May 2026 20:40:54 +0200 Subject: [PATCH 1/3] nuke: assumeutxo (commit 1 of 3) - drop user-facing surface Delete loadtxoutset and getchainstates RPCs, empty all chains' m_assumeutxo_data, drop feature_assumeutxo.py, tool_bitcoin_chainstate.py and utxo_snapshot fuzz target, delete the now-orphan snapshot-related unit tests. The snapshot machinery (ActivateSnapshot, PopulateAndValidateSnapshot, etc.) is still in place but unreachable from outside the binary. Removed in the next commit. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/kernel/chainparams.cpp | 90 -- src/rpc/blockchain.cpp | 171 +--- src/test/fuzz/rpc.cpp | 1 - src/test/fuzz/utxo_snapshot.cpp | 232 ----- src/test/validation_chainstate_tests.cpp | 74 -- .../validation_chainstatemanager_tests.cpp | 233 ----- src/test/validation_tests.cpp | 23 - test/functional/feature_assumeutxo.py | 797 ------------------ test/functional/tool_bitcoin_chainstate.py | 106 --- 9 files changed, 5 insertions(+), 1722 deletions(-) delete mode 100644 src/test/fuzz/utxo_snapshot.cpp delete mode 100755 test/functional/feature_assumeutxo.py delete mode 100755 test/functional/tool_bitcoin_chainstate.py diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 02c7ae9d2209..806c064ab42c 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -155,32 +155,6 @@ class CMainParams : public CChainParams { fDefaultConsistencyChecks = false; m_is_mockable_chain = false; - m_assumeutxo_data = { - { - .height = 840'000, - .hash_serialized = AssumeutxoHash{uint256{"a2a5521b1b5ab65f67818e5e8eccabb7171a517f9e2382208f77687310768f96"}}, - .m_chain_tx_count = 991032194, - .blockhash = uint256{"0000000000000000000320283a032748cef8227873ff4872689bf23f1cda83a5"}, - }, - { - .height = 880'000, - .hash_serialized = AssumeutxoHash{uint256{"dbd190983eaf433ef7c15f78a278ae42c00ef52e0fd2a54953782175fbadcea9"}}, - .m_chain_tx_count = 1145604538, - .blockhash = uint256{"000000000000000000010b17283c3c400507969a9c2afd1dcf2082ec5cca2880"}, - }, - { - .height = 910'000, - .hash_serialized = AssumeutxoHash{uint256{"4daf8a17b4902498c5787966a2b51c613acdab5df5db73f196fa59a4da2f1568"}}, - .m_chain_tx_count = 1226586151, - .blockhash = uint256{"0000000000000000000108970acb9522ffd516eae17acddcb1bd16469194a821"}, - }, - { - .height = 935'000, - .hash_serialized = AssumeutxoHash{uint256{"e4b90ef9eae834f56c4b64d2d50143cee10ad87994c614d7d04125e2a6025050"}}, - .m_chain_tx_count = 1305397408, - .blockhash = uint256{"0000000000000000000147034958af1652b2b91bba607beacc5e72a56f0fb5ee"}, - } - }; chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 00000000000000000000ccebd6d74d9194d8dcdc1d177c478e094bfad51ba5ac @@ -268,20 +242,6 @@ class CTestNetParams : public CChainParams { fDefaultConsistencyChecks = false; m_is_mockable_chain = false; - m_assumeutxo_data = { - { - .height = 2'500'000, - .hash_serialized = AssumeutxoHash{uint256{"f841584909f68e47897952345234e37fcd9128cd818f41ee6c3ca68db8071be7"}}, - .m_chain_tx_count = 66484552, - .blockhash = uint256{"0000000000000093bcb68c03a9a168ae252572d348a2eaeba2cdf9231d73206f"}, - }, - { - .height = 4'840'000, - .hash_serialized = AssumeutxoHash{uint256{"ce6bb677bb2ee9789c4a1c9d73e6683c53fc20e8fdbedbdaaf468982a0c8db2a"}}, - .m_chain_tx_count = 536078574, - .blockhash = uint256{"00000000000000f4971a7fb37fbdff89315b69a2e1920c467654a382f0d64786"}, - } - }; chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 000000007a61e4230b28ac5cb6b5e5a0130de37ac1faf2f8987d2fa6505b67f4 @@ -373,20 +333,6 @@ class CTestNet4Params : public CChainParams { fDefaultConsistencyChecks = false; m_is_mockable_chain = false; - m_assumeutxo_data = { - { - .height = 90'000, - .hash_serialized = AssumeutxoHash{uint256{"784fb5e98241de66fdd429f4392155c9e7db5c017148e66e8fdbc95746f8b9b5"}}, - .m_chain_tx_count = 11347043, - .blockhash = uint256{"0000000002ebe8bcda020e0dd6ccfbdfac531d2f6a81457191b99fc2df2dbe3b"}, - }, - { - .height = 120'000, - .hash_serialized = AssumeutxoHash{uint256{"10b05d05ad468d0971162e1b222a4aa66caca89da2bb2a93f8f37fb29c4794b0"}}, - .m_chain_tx_count = 14141057, - .blockhash = uint256{"000000000bd2317e51b3c5794981c35ba894ce27d3e772d5c39ecd9cbce01dc8"}, - } - }; chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 0000000002368b1e4ee27e2e85676ae6f9f9e69579b29093e9a82c170bf7cf8a @@ -486,20 +432,6 @@ class SigNetParams : public CChainParams { assert(consensus.hashGenesisBlock == uint256{"00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6"}); assert(genesis.hashMerkleRoot == uint256{"4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b"}); - m_assumeutxo_data = { - { - .height = 160'000, - .hash_serialized = AssumeutxoHash{uint256{"fe0a44309b74d6b5883d246cb419c6221bcccf0b308c9b59b7d70783dbdf928a"}}, - .m_chain_tx_count = 2289496, - .blockhash = uint256{"0000003ca3c99aff040f2563c2ad8f8ec88bd0fd6b8f0895cfaf1ef90353a62c"}, - }, - { - .height = 290'000, - .hash_serialized = AssumeutxoHash{uint256{"97267e000b4b876800167e71b9123f1529d13b14308abec2888bbd2160d14545"}}, - .m_chain_tx_count = 28547497, - .blockhash = uint256{"0000000577f2741bb30cd9d39d6d71b023afbeb9764f6260786a97969d5c9ac0"}, - } - }; base58Prefixes[PUBKEY_ADDRESS] = std::vector(1,111); base58Prefixes[SCRIPT_ADDRESS] = std::vector(1,196); @@ -604,28 +536,6 @@ class CRegTestParams : public CChainParams fDefaultConsistencyChecks = true; m_is_mockable_chain = true; - m_assumeutxo_data = { - { // For use by unit tests - .height = 110, - .hash_serialized = AssumeutxoHash{uint256{"b952555c8ab81fec46f3d4253b7af256d766ceb39fb7752b9d18cdf4a0141327"}}, - .m_chain_tx_count = 111, - .blockhash = uint256{"6affe030b7965ab538f820a56ef56c8149b7dc1d1c144af57113be080db7c397"}, - }, - { - // For use by fuzz target src/test/fuzz/utxo_snapshot.cpp - .height = 200, - .hash_serialized = AssumeutxoHash{uint256{"17dcc016d188d16068907cdeb38b75691a118d43053b8cd6a25969419381d13a"}}, - .m_chain_tx_count = 201, - .blockhash = uint256{"385901ccbd69dff6bbd00065d01fb8a9e464dede7cfe0372443884f9b1dcf6b9"}, - }, - { - // For use by test/functional/feature_assumeutxo.py and test/functional/tool_bitcoin_chainstate.py - .height = 299, - .hash_serialized = AssumeutxoHash{uint256{"d2b051ff5e8eef46520350776f4100dd710a63447a8e01d917e92e79751a63e2"}}, - .m_chain_tx_count = 334, - .blockhash = uint256{"7cc695046fec709f8c9394b6f928f81e81fd3ac20977bb68760fa1faa7916ea2"}, - }, - }; chainTxData = ChainTxData{ .nTime = 0, diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 35a94a2fbda5..5c053300e70d 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1715,8 +1715,7 @@ static RPCMethod getchaintxstats() { {RPCResult::Type::NUM_TIME, "time", "The timestamp for the final block in the window, expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::NUM, "txcount", /*optional=*/true, - "The total number of transactions in the chain up to that point, if known. " - "It may be unknown when using assumeutxo."}, + "The total number of transactions in the chain up to that point, if known."}, {RPCResult::Type::STR_HEX, "window_final_block_hash", "The hash of the final block in the window"}, {RPCResult::Type::NUM, "window_final_block_height", "The height of the final block in the window."}, {RPCResult::Type::NUM, "window_block_count", "Size of the window in number of blocks"}, @@ -2637,12 +2636,12 @@ static RPCMethod dumptxoutset() { return RPCMethod{ "dumptxoutset", - "Write the serialized UTXO set to a file. This can be used in loadtxoutset afterwards if this snapshot height is supported in the chainparams as well.\n" + "Write the serialized UTXO set to a file.\n" "This creates a temporary UTXO database when rolling back, keeping the main chain intact. Should the node experience an unclean shutdown the temporary database may need to be removed from the datadir manually.\n" "For deep rollbacks, make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0) as it may take several minutes.", { {"path", RPCArg::Type::STR, RPCArg::Optional::NO, "Path to the output file. If relative, will be prefixed by datadir."}, - {"type", RPCArg::Type::STR, RPCArg::Default(""), "The type of snapshot to create. Can be \"latest\" to create a snapshot of the current UTXO set or \"rollback\" to temporarily roll back the state of the node to a historical block before creating the snapshot of a historical UTXO set. This parameter can be omitted if a separate \"rollback\" named parameter is specified indicating the height or hash of a specific historical block. If \"rollback\" is specified and separate \"rollback\" named parameter is not specified, this will roll back to the latest valid snapshot block that can currently be loaded with loadtxoutset."}, + {"type", RPCArg::Type::STR, RPCArg::Default(""), "The type of snapshot to create. Can be \"latest\" to create a snapshot of the current UTXO set or \"rollback\" to temporarily roll back the state of the node to a historical block before creating the snapshot of a historical UTXO set. This parameter can be omitted if a separate \"rollback\" named parameter is specified indicating the height or hash of a specific historical block."}, {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"rollback", RPCArg::Type::NUM, RPCArg::Optional::OMITTED, @@ -2665,7 +2664,6 @@ static RPCMethod dumptxoutset() }, RPCExamples{ HelpExampleCli("-rpcclienttimeout=0 dumptxoutset", "utxo.dat latest") + - HelpExampleCli("-rpcclienttimeout=0 dumptxoutset", "utxo.dat rollback") + HelpExampleCli("-rpcclienttimeout=0 -named dumptxoutset", R"(utxo.dat rollback=853456)") + HelpExampleCli("-rpcclienttimeout=0 -named dumptxoutset", R"(utxo.dat rollback=853456 in_memory=true)") }, @@ -2681,13 +2679,10 @@ static RPCMethod dumptxoutset() throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid snapshot type \"%s\" specified with rollback option", snapshot_type)); } target_index = ParseHashOrHeight(options["rollback"], *node.chainman); - } else if (snapshot_type == "rollback") { - auto snapshot_heights = node.chainman->GetParams().GetAvailableSnapshotHeights(); - CHECK_NONFATAL(snapshot_heights.size() > 0); - auto max_height = std::max_element(snapshot_heights.begin(), snapshot_heights.end()); - target_index = ParseHashOrHeight(*max_height, *node.chainman); } else if (snapshot_type == "latest") { target_index = tip; + } else if (snapshot_type == "rollback") { + throw JSONRPCError(RPC_INVALID_PARAMETER, "When specifying \"rollback\", a rollback height or hash must be provided via the named \"rollback\" parameter"); } else { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid snapshot type \"%s\" specified. Please specify \"rollback\" or \"latest\"", snapshot_type)); } @@ -3047,160 +3042,6 @@ UniValue CreateUTXOSnapshot( node.rpc_interruption_point); } -static RPCMethod loadtxoutset() -{ - return RPCMethod{ - "loadtxoutset", - "Load the serialized UTXO set from a file.\n" - "Once this snapshot is loaded, its contents will be " - "deserialized into a second chainstate data structure, which is then used to sync to " - "the network's tip. " - "Meanwhile, the original chainstate will complete the initial block download process in " - "the background, eventually validating up to the block that the snapshot is based upon.\n\n" - - "The result is a usable bitcoind instance that is current with the network tip in a " - "matter of minutes rather than hours. UTXO snapshot are typically obtained from " - "third-party sources (HTTP, torrent, etc.) which is reasonable since their " - "contents are always checked by hash.\n\n" - - "You can find more information on this process in the `assumeutxo` design " - "document ().", - { - {"path", - RPCArg::Type::STR, - RPCArg::Optional::NO, - "path to the snapshot file. If relative, will be prefixed by datadir."}, - }, - RPCResult{ - RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "coins_loaded", "the number of coins loaded from the snapshot"}, - {RPCResult::Type::STR_HEX, "tip_hash", "the hash of the base of the snapshot"}, - {RPCResult::Type::NUM, "base_height", "the height of the base of the snapshot"}, - {RPCResult::Type::STR, "path", "the absolute path that the snapshot was loaded from"}, - } - }, - RPCExamples{ - HelpExampleCli("-rpcclienttimeout=0 loadtxoutset", "utxo.dat") - }, - [](const RPCMethod& self, const JSONRPCRequest& request) -> UniValue -{ - NodeContext& node = EnsureAnyNodeContext(request.context); - ChainstateManager& chainman = EnsureChainman(node); - const fs::path path{AbsPathForConfigVal(EnsureArgsman(node), fs::u8path(self.Arg("path")))}; - - FILE* file{fsbridge::fopen(path, "rb")}; - AutoFile afile{file}; - if (afile.IsNull()) { - throw JSONRPCError( - RPC_INVALID_PARAMETER, - "Couldn't open file " + path.utf8string() + " for reading."); - } - - SnapshotMetadata metadata{chainman.GetParams().MessageStart()}; - try { - afile >> metadata; - } catch (const std::ios_base::failure& e) { - throw JSONRPCError(RPC_DESERIALIZATION_ERROR, strprintf("Unable to parse metadata: %s", e.what())); - } - - auto activation_result{chainman.ActivateSnapshot(afile, metadata, false)}; - if (!activation_result) { - throw JSONRPCError(RPC_INTERNAL_ERROR, strprintf("Unable to load UTXO snapshot: %s. (%s)", util::ErrorString(activation_result).original, path.utf8string())); - } - - // Because we can't provide historical blocks during tip or background sync. - // Update local services to reflect we are a limited peer until we are fully sync. - node.connman->RemoveLocalServices(NODE_NETWORK); - // Setting the limited state is usually redundant because the node can always - // provide the last 288 blocks, but it doesn't hurt to set it. - node.connman->AddLocalServices(NODE_NETWORK_LIMITED); - - CBlockIndex& snapshot_index{*CHECK_NONFATAL(*activation_result)}; - - UniValue result(UniValue::VOBJ); - result.pushKV("coins_loaded", metadata.m_coins_count); - result.pushKV("tip_hash", snapshot_index.GetBlockHash().ToString()); - result.pushKV("base_height", snapshot_index.nHeight); - result.pushKV("path", fs::PathToString(path)); - return result; -}, - }; -} - -const std::vector RPCHelpForChainstate{ - {RPCResult::Type::NUM, "blocks", "number of blocks in this chainstate"}, - {RPCResult::Type::STR_HEX, "bestblockhash", "blockhash of the tip"}, - {RPCResult::Type::STR_HEX, "bits", "nBits: compact representation of the block difficulty target"}, - {RPCResult::Type::STR_HEX, "target", "The difficulty target"}, - {RPCResult::Type::NUM, "difficulty", "difficulty of the tip"}, - {RPCResult::Type::NUM, "verificationprogress", "progress towards the network tip"}, - {RPCResult::Type::STR_HEX, "snapshot_blockhash", /*optional=*/true, "the base block of the snapshot this chainstate is based on, if any"}, - {RPCResult::Type::NUM, "coins_db_cache_bytes", "size of the coinsdb cache"}, - {RPCResult::Type::NUM, "coins_tip_cache_bytes", "size of the coinstip cache"}, - {RPCResult::Type::BOOL, "validated", "whether the chainstate is fully validated. True if all blocks in the chainstate were validated, false if the chain is based on a snapshot and the snapshot has not yet been validated."}, -}; - -static RPCMethod getchainstates() -{ -return RPCMethod{ - "getchainstates", - "Return information about chainstates.\n", - {}, - RPCResult{ - RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::NUM, "headers", "the number of headers seen so far"}, - {RPCResult::Type::ARR, "chainstates", "list of the chainstates ordered by work, with the most-work (active) chainstate last", {{RPCResult::Type::OBJ, "", "", RPCHelpForChainstate},}}, - } - }, - RPCExamples{ - HelpExampleCli("getchainstates", "") - + HelpExampleRpc("getchainstates", "") - }, - [](const RPCMethod& self, const JSONRPCRequest& request) -> UniValue -{ - LOCK(cs_main); - UniValue obj(UniValue::VOBJ); - - ChainstateManager& chainman = EnsureAnyChainman(request.context); - - auto make_chain_data = [&](const Chainstate& cs) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - AssertLockHeld(::cs_main); - UniValue data(UniValue::VOBJ); - if (!cs.m_chain.Tip()) { - return data; - } - const CChain& chain = cs.m_chain; - const CBlockIndex* tip = chain.Tip(); - - data.pushKV("blocks", chain.Height()); - data.pushKV("bestblockhash", tip->GetBlockHash().GetHex()); - data.pushKV("bits", strprintf("%08x", tip->nBits)); - data.pushKV("target", GetTarget(*tip, chainman.GetConsensus().powLimit).GetHex()); - data.pushKV("difficulty", GetDifficulty(*tip)); - data.pushKV("verificationprogress", chainman.GuessVerificationProgress(tip)); - data.pushKV("coins_db_cache_bytes", cs.m_coinsdb_cache_size_bytes); - data.pushKV("coins_tip_cache_bytes", cs.m_coinstip_cache_size_bytes); - if (cs.m_from_snapshot_blockhash) { - data.pushKV("snapshot_blockhash", cs.m_from_snapshot_blockhash->ToString()); - } - data.pushKV("validated", cs.m_assumeutxo == Assumeutxo::VALIDATED); - return data; - }; - - obj.pushKV("headers", chainman.m_best_header ? chainman.m_best_header->nHeight : -1); - UniValue obj_chainstates{UniValue::VARR}; - if (const Chainstate * cs{chainman.HistoricalChainstate()}) { - obj_chainstates.push_back(make_chain_data(*cs)); - } - obj_chainstates.push_back(make_chain_data(chainman.CurrentChainstate())); - obj.pushKV("chainstates", std::move(obj_chainstates)); - return obj; -} - }; -} - - void RegisterBlockchainRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -3224,8 +3065,6 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &scantxoutset}, {"blockchain", &getdescriptoractivity}, {"blockchain", &dumptxoutset}, - {"blockchain", &loadtxoutset}, - {"blockchain", &getchainstates}, {"hidden", &invalidateblock}, {"hidden", &reconsiderblock}, {"blockchain", &waitfornewblock}, diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 42c02e7aae74..fe05b9361b5a 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -82,7 +82,6 @@ const std::vector RPC_COMMANDS_NOT_SAFE_FOR_FUZZING{ "generatetodescriptor", // avoid prohibitively slow execution (when `nblocks` is large) "gettxoutproof", // avoid prohibitively slow execution "importmempool", // avoid reading from disk - "loadtxoutset", // avoid reading from disk "savemempool", // disabled as a precautionary measure: may take a file path argument in the future "setban", // avoid DNS lookups "stop", // avoid shutdown state diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp deleted file mode 100644 index ed286865d087..000000000000 --- a/src/test/fuzz/utxo_snapshot.cpp +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright (c) 2021-present The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -using node::SnapshotMetadata; - -namespace { - -const std::vector>* g_chain; -TestingSetup* g_setup{nullptr}; - -/** Sanity check the assumeutxo values hardcoded in chainparams for the fuzz target. */ -void sanity_check_snapshot() -{ - Assert(g_chain && g_setup == nullptr); - - // Create a temporary chainstate manager to connect the chain to. - const auto tmp_setup{MakeNoLogFileContext(ChainType::REGTEST, TestOpts{.setup_net = false})}; - const auto& node{tmp_setup->m_node}; - for (auto& block: *g_chain) { - ProcessBlock(node, block); - } - - // Connect the chain to the tmp chainman and sanity check the chainparams snapshot values. - LOCK(cs_main); - auto& cs{node.chainman->ActiveChainstate()}; - cs.ForceFlushStateToDisk(/*wipe_cache=*/false); - const auto stats{*Assert(kernel::ComputeUTXOStats(kernel::CoinStatsHashType::HASH_SERIALIZED, &cs.CoinsDB(), node.chainman->m_blockman))}; - const auto cp_au_data{*Assert(node.chainman->GetParams().AssumeutxoForHeight(2 * COINBASE_MATURITY))}; - Assert(stats.nHeight == cp_au_data.height); - Assert(stats.nTransactions + 1 == cp_au_data.m_chain_tx_count); // +1 for the genesis tx. - Assert(stats.hashBlock == cp_au_data.blockhash); - Assert(AssumeutxoHash{stats.hashSerialized} == cp_au_data.hash_serialized); -} - -template -void initialize_chain() -{ - const auto params{CreateChainParams(ArgsManager{}, ChainType::REGTEST)}; - static const auto chain{CreateBlockChain(2 * COINBASE_MATURITY, *params)}; - g_chain = &chain; - SetMockTime(chain.back()->Time()); - - // Make sure we can generate a valid snapshot. - sanity_check_snapshot(); - - static const auto setup{ - MakeNoLogFileContext(ChainType::REGTEST, - TestOpts{ - .setup_net = false, - .setup_validation_interface = false, - .min_validation_cache = true, - }), - }; - if constexpr (INVALID) { - auto& chainman{*setup->m_node.chainman}; - for (const auto& block : chain) { - BlockValidationState dummy; - bool processed{chainman.ProcessNewBlockHeaders({{*block}}, true, dummy)}; - Assert(processed); - const auto* index{WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(block->GetHash()))}; - Assert(index); - } - } - g_setup = setup.get(); -} - -template -void utxo_snapshot_fuzz(FuzzBufferType buffer) -{ - SeedRandomStateForTest(SeedRand::ZEROS); - FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - NodeClockContext clock_ctx{ConsumeTime(fuzzed_data_provider, /*min=*/1296688602)}; // regtest genesis block timestamp - auto& setup{*g_setup}; - bool dirty_chainman{false}; // Reuse the global chainman, but reset it when it is dirty - auto& chainman{*setup.m_node.chainman}; - - const auto snapshot_path = gArgs.GetDataDirNet() / "fuzzed_snapshot.dat"; - - Assert(!chainman.ActiveChainstate().m_from_snapshot_blockhash); - - { - AutoFile outfile{fsbridge::fopen(snapshot_path, "wb")}; - // Metadata - if (fuzzed_data_provider.ConsumeBool()) { - std::vector metadata{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; - outfile << std::span{metadata}; - } else { - auto msg_start = chainman.GetParams().MessageStart(); - int base_blockheight{fuzzed_data_provider.ConsumeIntegralInRange(1, 2 * COINBASE_MATURITY)}; - uint256 base_blockhash{g_chain->at(base_blockheight - 1)->GetHash()}; - uint64_t m_coins_count{fuzzed_data_provider.ConsumeIntegralInRange(1, 3 * COINBASE_MATURITY)}; - SnapshotMetadata metadata{msg_start, base_blockhash, m_coins_count}; - outfile << metadata; - } - // Coins - if (fuzzed_data_provider.ConsumeBool()) { - std::vector file_data{ConsumeRandomLengthByteVector(fuzzed_data_provider)}; - outfile << std::span{file_data}; - } else { - int height{1}; - for (const auto& block : *g_chain) { - auto coinbase{block->vtx.at(0)}; - outfile << coinbase->GetHash(); - WriteCompactSize(outfile, 1); // number of coins for the hash - WriteCompactSize(outfile, 0); // index of coin - outfile << Coin(coinbase->vout[0], height, /*fCoinBaseIn=*/true); - height++; - } - } - if constexpr (INVALID) { - // Append an invalid coin to ensure invalidity. This error will be - // detected late in PopulateAndValidateSnapshot, and allows the - // INVALID fuzz target to reach more potential code coverage. - const auto& coinbase{g_chain->back()->vtx.back()}; - outfile << coinbase->GetHash(); - WriteCompactSize(outfile, 1); // number of coins for the hash - WriteCompactSize(outfile, 999); // index of coin - outfile << Coin{coinbase->vout[0], /*nHeightIn=*/999, /*fCoinBaseIn=*/false}; - } - assert(outfile.fclose() == 0); - } - - const auto ActivateFuzzedSnapshot{[&] { - AutoFile infile{fsbridge::fopen(snapshot_path, "rb")}; - auto msg_start = chainman.GetParams().MessageStart(); - SnapshotMetadata metadata{msg_start}; - try { - infile >> metadata; - } catch (const std::ios_base::failure&) { - return false; - } - return !!chainman.ActivateSnapshot(infile, metadata, /*in_memory=*/true); - }}; - - if (fuzzed_data_provider.ConsumeBool()) { - // Consume the bool, but skip the code for the INVALID fuzz target - if constexpr (!INVALID) { - for (const auto& block : *g_chain) { - BlockValidationState dummy; - bool processed{chainman.ProcessNewBlockHeaders({{*block}}, true, dummy)}; - Assert(processed); - const auto* index{WITH_LOCK(::cs_main, return chainman.m_blockman.LookupBlockIndex(block->GetHash()))}; - Assert(index); - } - dirty_chainman = true; - } - } - - if (ActivateFuzzedSnapshot()) { - LOCK(::cs_main); - Assert(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); - const auto& coinscache{chainman.ActiveChainstate().CoinsTip()}; - for (const auto& block : *g_chain) { - Assert(coinscache.HaveCoin(COutPoint{block->vtx.at(0)->GetHash(), 0})); - const auto* index{chainman.m_blockman.LookupBlockIndex(block->GetHash())}; - Assert(index); - Assert(index->nTx == 0); - if (index->nHeight == chainman.ActiveChainstate().SnapshotBase()->nHeight) { - auto params{chainman.GetParams().AssumeutxoForHeight(index->nHeight)}; - Assert(params.has_value()); - Assert(params.value().m_chain_tx_count == index->m_chain_tx_count); - } else { - Assert(index->m_chain_tx_count == 0); - } - } - Assert(g_chain->size() == coinscache.GetCacheSize()); - dirty_chainman = true; - } else { - Assert(!chainman.ActiveChainstate().m_from_snapshot_blockhash); - } - // Snapshot should refuse to load a second time regardless of validity - Assert(!ActivateFuzzedSnapshot()); - if constexpr (INVALID) { - // Activating the snapshot, or any other action that makes the chainman - // "dirty" can and must not happen for the INVALID fuzz target - Assert(!dirty_chainman); - } - if (dirty_chainman) { - setup.m_node.chainman.reset(); - setup.m_make_chainman(); - setup.LoadVerifyActivateChainstate(); - } -} - -// There are two fuzz targets: -// -// The target 'utxo_snapshot', which allows valid snapshots, but is slow, -// because it has to reset the chainstate manager on almost all fuzz inputs. -// Otherwise, a dirty header tree or dirty chainstate could leak from one fuzz -// input execution into the next, which makes execution non-deterministic. -// -// The target 'utxo_snapshot_invalid', which is fast and does not require any -// expensive state to be reset. -FUZZ_TARGET(utxo_snapshot /*valid*/, .init = initialize_chain) { utxo_snapshot_fuzz(buffer); } -FUZZ_TARGET(utxo_snapshot_invalid, .init = initialize_chain) { utxo_snapshot_fuzz(buffer); } - -} // namespace diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 38a04d8e46ce..004a71325dc8 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -91,79 +91,5 @@ BOOST_FIXTURE_TEST_CASE(connect_tip_does_not_cache_inputs_on_failed_connect, Tes BOOST_CHECK(!chainstate.CoinsTip().HaveCoinInCache(outpoint)); // input not cached } -//! Test UpdateTip behavior for both active and background chainstates. -//! -//! When run on the background chainstate, UpdateTip should do a subset -//! of what it does for the active chainstate. -BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) -{ - ChainstateManager& chainman = *Assert(m_node.chainman); - const auto get_notify_tip{[&]() { - LOCK(m_node.notifications->m_tip_block_mutex); - BOOST_REQUIRE(m_node.notifications->TipBlock()); - return *m_node.notifications->TipBlock(); - }}; - uint256 curr_tip = get_notify_tip(); - - // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can - // be found. - mineBlocks(10); - - // After adding some blocks to the tip, best block should have changed. - BOOST_CHECK(get_notify_tip() != curr_tip); - - // Grab block 1 from disk; we'll add it to the background chain later. - std::shared_ptr pblockone = std::make_shared(); - { - LOCK(::cs_main); - chainman.m_blockman.ReadBlock(*pblockone, *chainman.ActiveChain()[1]); - } - - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot( - this, NoMalleation, /*reset_chainstate=*/ true)); - - // Ensure our active chain is the snapshot chainstate. - BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.CurrentChainstate().m_from_snapshot_blockhash)); - - curr_tip = get_notify_tip(); - - // Mine a new block on top of the activated snapshot chainstate. - mineBlocks(1); // Defined in TestChain100Setup. - - // After adding some blocks to the snapshot tip, best block should have changed. - BOOST_CHECK(get_notify_tip() != curr_tip); - - curr_tip = get_notify_tip(); - - Chainstate& background_cs{*Assert(WITH_LOCK(::cs_main, return chainman.HistoricalChainstate()))}; - - // Append the first block to the background chain. - BlockValidationState state; - CBlockIndex* pindex = nullptr; - const CChainParams& chainparams = Params(); - bool newblock = false; - - // TODO: much of this is inlined from ProcessNewBlock(); just reuse PNB() - // once it is changed to support multiple chainstates. - { - LOCK(::cs_main); - bool checked = CheckBlock(*pblockone, state, chainparams.GetConsensus()); - BOOST_CHECK(checked); - bool accepted = chainman.AcceptBlock( - pblockone, state, &pindex, true, nullptr, &newblock, true); - BOOST_CHECK(accepted); - } - - // UpdateTip is called here - bool block_added = background_cs.ActivateBestChain(state, pblockone); - - // Ensure tip is as expected - BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), pblockone->GetHash()); - - // get_notify_tip() should be unchanged after adding a block to the background - // validation chain. - BOOST_CHECK(block_added); - BOOST_CHECK_EQUAL(curr_tip, get_notify_tip()); -} BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 9818b51e51b9..8db76e042715 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -451,12 +451,6 @@ struct SnapshotTestSetup : TestChain100Setup { } }; -//! Test basic snapshot activation. -BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) -{ - this->SetupSnapshot(); -} - //! Test LoadBlockIndex behavior when multiple chainstates are in use. //! //! - First, verify that setBlockIndexCandidates is as expected when using a single, @@ -707,233 +701,6 @@ BOOST_FIXTURE_TEST_CASE(invalidate_block_and_reconsider_fork, TestChain100Setup) } } -//! Ensure that snapshot chainstate can be loaded when found on disk after a -//! restart, and that new blocks can be connected to both chainstates. -BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) -{ - ChainstateManager& chainman = *Assert(m_node.chainman); - Chainstate& bg_chainstate = chainman.ActiveChainstate(); - - this->SetupSnapshot(); - - fs::path snapshot_chainstate_dir = *node::FindAssumeutxoChainstateDir(chainman.m_options.datadir); - BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); - BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); - - BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.CurrentChainstate().m_from_snapshot_blockhash)); - const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(), - return chainman.ActiveTip()->GetBlockHash()); - - BOOST_CHECK_EQUAL(WITH_LOCK(chainman.GetMutex(), return chainman.m_chainstates.size()), 2); - - // "Rewind" the background chainstate so that its tip is not at the - // base block of the snapshot - this is so after simulating a node restart, - // it will initialize instead of attempting to complete validation. - // - // Note that this is not a realistic use of DisconnectTip(). - DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_BYTES}; - BlockValidationState unused_state; - { - LOCK2(::cs_main, bg_chainstate.MempoolMutex()); - BOOST_CHECK(bg_chainstate.DisconnectTip(unused_state, &unused_pool)); - unused_pool.clear(); // to avoid queuedTx assertion errors on teardown - } - BOOST_CHECK_EQUAL(bg_chainstate.m_chain.Height(), 109); - - // Test that simulating a shutdown (resetting ChainstateManager) and then performing - // chainstate reinitializing successfully reloads both chainstates. - ChainstateManager& chainman_restarted = this->SimulateNodeRestart(); - - BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); - - // This call reinitializes the chainstates. - this->LoadVerifyActivateChainstate(); - - { - LOCK(chainman_restarted.GetMutex()); - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates.size(), 2); - // Background chainstate has height of 109 not 110 here due to a quirk - // of the LoadVerifyActivate only calling ActivateBestChain on one - // chainstate. The height would be 110 after a real restart, but it's - // fine for this test which is focused on the snapshot chainstate. - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates[0]->m_chain.Height(), 109); - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates[1]->m_chain.Height(), 210); - - BOOST_CHECK(chainman_restarted.CurrentChainstate().m_from_snapshot_blockhash); - BOOST_CHECK(chainman_restarted.CurrentChainstate().m_assumeutxo == Assumeutxo::UNVALIDATED); - - BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); - BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); - BOOST_CHECK_EQUAL(chainman_restarted.HistoricalChainstate()->m_chain.Height(), 109); - } - - BOOST_TEST_MESSAGE( - "Ensure we can mine blocks on top of the initialized snapshot chainstate"); - mineBlocks(10); - { - LOCK(chainman_restarted.GetMutex()); - BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); - - // Background chainstate should be unaware of new blocks on the snapshot - // chainstate, but the block disconnected above is now reattached. - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates.size(), 2); - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates[0]->m_chain.Height(), 110); - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates[1]->m_chain.Height(), 220); - BOOST_CHECK_EQUAL(chainman_restarted.HistoricalChainstate(), nullptr); - } -} - -BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion, SnapshotTestSetup) -{ - this->SetupSnapshot(); - - ChainstateManager& chainman = *Assert(m_node.chainman); - Chainstate& active_cs = chainman.ActiveChainstate(); - Chainstate& validated_cs{*Assert(WITH_LOCK(cs_main, return chainman.HistoricalChainstate()))}; - auto tip_cache_before_complete = active_cs.m_coinstip_cache_size_bytes; - auto db_cache_before_complete = active_cs.m_coinsdb_cache_size_bytes; - - SnapshotCompletionResult res; - m_node.notifications->m_shutdown_on_fatal_error = false; - - fs::path snapshot_chainstate_dir = *node::FindAssumeutxoChainstateDir(chainman.m_options.datadir); - BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); - BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); - - BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.CurrentChainstate().m_from_snapshot_blockhash)); - const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(), - return chainman.ActiveTip()->GetBlockHash()); - - res = WITH_LOCK(::cs_main, return chainman.MaybeValidateSnapshot(validated_cs, active_cs)); - BOOST_CHECK_EQUAL(res, SnapshotCompletionResult::SUCCESS); - - BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.CurrentChainstate().m_assumeutxo == Assumeutxo::VALIDATED)); - BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.CurrentChainstate().m_from_snapshot_blockhash)); - BOOST_CHECK_EQUAL(WITH_LOCK(chainman.GetMutex(), return chainman.HistoricalChainstate()), nullptr); - - // Cache should have been rebalanced and reallocated to the "only" remaining - // chainstate. - BOOST_CHECK(active_cs.m_coinstip_cache_size_bytes > tip_cache_before_complete); - BOOST_CHECK(active_cs.m_coinsdb_cache_size_bytes > db_cache_before_complete); - - // Trying completion again should return false. - res = WITH_LOCK(::cs_main, return chainman.MaybeValidateSnapshot(validated_cs, active_cs)); - BOOST_CHECK_EQUAL(res, SnapshotCompletionResult::SKIPPED); - - // The invalid snapshot path should not have been used. - fs::path snapshot_invalid_dir = gArgs.GetDataDirNet() / "chainstate_snapshot_INVALID"; - BOOST_CHECK(!fs::exists(snapshot_invalid_dir)); - // chainstate_snapshot should still exist. - BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); - - // Test that simulating a shutdown (resetting ChainstateManager) and then performing - // chainstate reinitializing successfully cleans up the background-validation - // chainstate data, and we end up with a single chainstate that is at tip. - ChainstateManager& chainman_restarted = this->SimulateNodeRestart(); - - BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); - - // This call reinitializes the chainstates, and should clean up the now unnecessary - // background-validation leveldb contents. - this->LoadVerifyActivateChainstate(); - - BOOST_CHECK(!fs::exists(snapshot_invalid_dir)); - // chainstate_snapshot should now *not* exist. - BOOST_CHECK(!fs::exists(snapshot_chainstate_dir)); - - const Chainstate& active_cs2 = chainman_restarted.ActiveChainstate(); - - { - LOCK(chainman_restarted.GetMutex()); - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates.size(), 1); - BOOST_CHECK(!chainman_restarted.CurrentChainstate().m_from_snapshot_blockhash); - BOOST_CHECK(active_cs2.m_coinstip_cache_size_bytes > tip_cache_before_complete); - BOOST_CHECK(active_cs2.m_coinsdb_cache_size_bytes > db_cache_before_complete); - - BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); - BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); - } - - BOOST_TEST_MESSAGE( - "Ensure we can mine blocks on top of the \"new\" IBD chainstate"); - mineBlocks(10); - { - LOCK(chainman_restarted.GetMutex()); - BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); - } -} - -BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion_hash_mismatch, SnapshotTestSetup) -{ - auto chainstates = this->SetupSnapshot(); - Chainstate& validation_chainstate = *std::get<0>(chainstates); - Chainstate& unvalidated_cs = *std::get<1>(chainstates); - ChainstateManager& chainman = *Assert(m_node.chainman); - SnapshotCompletionResult res; - m_node.notifications->m_shutdown_on_fatal_error = false; - - // Test tampering with the IBD UTXO set with an extra coin to ensure it causes - // snapshot completion to fail. - CCoinsViewCache& ibd_coins = WITH_LOCK(::cs_main, - return validation_chainstate.CoinsTip()); - Coin badcoin; - badcoin.out.nValue = m_rng.rand32(); - badcoin.nHeight = 1; - badcoin.out.scriptPubKey.assign(m_rng.randbits(6), 0); - Txid txid = Txid::FromUint256(m_rng.rand256()); - ibd_coins.AddCoin(COutPoint(txid, 0), std::move(badcoin), false); - - fs::path snapshot_chainstate_dir = gArgs.GetDataDirNet() / "chainstate_snapshot"; - BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); - - { - ASSERT_DEBUG_LOG("failed to validate the -assumeutxo snapshot state"); - res = WITH_LOCK(::cs_main, return chainman.MaybeValidateSnapshot(validation_chainstate, unvalidated_cs)); - BOOST_CHECK_EQUAL(res, SnapshotCompletionResult::HASH_MISMATCH); - } - - { - LOCK(chainman.GetMutex()); - BOOST_CHECK_EQUAL(chainman.m_chainstates.size(), 2); - BOOST_CHECK(chainman.m_chainstates[0]->m_assumeutxo == Assumeutxo::VALIDATED); - BOOST_CHECK(!chainman.m_chainstates[0]->SnapshotBase()); - BOOST_CHECK(chainman.m_chainstates[1]->m_assumeutxo == Assumeutxo::INVALID); - BOOST_CHECK(chainman.m_chainstates[1]->SnapshotBase()); - } - - fs::path snapshot_invalid_dir = gArgs.GetDataDirNet() / "chainstate_snapshot_INVALID"; - BOOST_CHECK(fs::exists(snapshot_invalid_dir)); - - // Test that simulating a shutdown (resetting ChainstateManager) and then performing - // chainstate reinitializing successfully loads only the fully-validated - // chainstate data, and we end up with a single chainstate that is at tip. - ChainstateManager& chainman_restarted = this->SimulateNodeRestart(); - - BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); - - // This call reinitializes the chainstates, and should clean up the now unnecessary - // background-validation leveldb contents. - this->LoadVerifyActivateChainstate(); - - BOOST_CHECK(fs::exists(snapshot_invalid_dir)); - BOOST_CHECK(!fs::exists(snapshot_chainstate_dir)); - - { - LOCK(::cs_main); - BOOST_CHECK_EQUAL(chainman_restarted.m_chainstates.size(), 1); - BOOST_CHECK(!chainman_restarted.CurrentChainstate().m_from_snapshot_blockhash); - BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); - } - - BOOST_TEST_MESSAGE( - "Ensure we can mine blocks on top of the \"new\" IBD chainstate"); - mineBlocks(10); - { - LOCK(::cs_main); - BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); - } -} - /** Helper function to parse args into args_man and return the result of applying them to opts */ template util::Result SetOptsFromArgs(ArgsManager& args_man, Options opts, diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index b1e204f17413..07b28b9cd4b4 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -127,29 +127,6 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) BOOST_CHECK(!CheckSignetBlockSolution(block, signet_params->GetConsensus())); } -//! Test retrieval of valid assumeutxo values. -BOOST_AUTO_TEST_CASE(test_assumeutxo) -{ - const auto params = CreateChainParams(*m_node.args, ChainType::REGTEST); - - // These heights don't have assumeutxo configurations associated, per the contents - // of kernel/chainparams.cpp. - std::vector bad_heights{0, 100, 111, 115, 209, 211}; - - for (auto empty : bad_heights) { - const auto out = params->AssumeutxoForHeight(empty); - BOOST_CHECK(!out); - } - - const auto out110 = *params->AssumeutxoForHeight(110); - BOOST_CHECK_EQUAL(out110.hash_serialized.ToString(), "b952555c8ab81fec46f3d4253b7af256d766ceb39fb7752b9d18cdf4a0141327"); - BOOST_CHECK_EQUAL(out110.m_chain_tx_count, 111U); - - const auto out110_2 = *params->AssumeutxoForBlockhash(uint256{"6affe030b7965ab538f820a56ef56c8149b7dc1d1c144af57113be080db7c397"}); - BOOST_CHECK_EQUAL(out110_2.hash_serialized.ToString(), "b952555c8ab81fec46f3d4253b7af256d766ceb39fb7752b9d18cdf4a0141327"); - BOOST_CHECK_EQUAL(out110_2.m_chain_tx_count, 111U); -} - BOOST_AUTO_TEST_CASE(block_malleation) { // Test utilities that calls `IsBlockMutated` and then clears the validity diff --git a/test/functional/feature_assumeutxo.py b/test/functional/feature_assumeutxo.py deleted file mode 100755 index 9e0c91407592..000000000000 --- a/test/functional/feature_assumeutxo.py +++ /dev/null @@ -1,797 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2021-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test for assumeutxo, a means of quickly bootstrapping a node using -a serialized version of the UTXO set at a certain height, which corresponds -to a hash that has been compiled into bitcoind. - -The assumeutxo value generated and used here is committed to in -`CRegTestParams::m_assumeutxo_data` in `src/kernel/chainparams.cpp`. -""" -import contextlib - -from dataclasses import dataclass -from test_framework.blocktools import ( - create_block, -) -from test_framework.compressor import ( - compress_amount, -) -from test_framework.messages import ( - CBlockHeader, - from_hex, - MAGIC_BYTES, - MAX_MONEY, - msg_headers, - ser_varint, - tx_from_hex, -) -from test_framework.p2p import ( - P2PInterface, -) -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import ( - assert_approx, - assert_equal, - assert_not_equal, - assert_raises_rpc_error, - dumb_sync_blocks, - ensure_for, - sha256sum_file, - try_rpc, -) -from test_framework.wallet import ( - getnewdestination, - MiniWallet, -) -from test_framework.blocktools import ( - REGTEST_N_BITS, - REGTEST_TARGET, - nbits_str, - target_str, -) - -START_HEIGHT = 199 -SNAPSHOT_BASE_HEIGHT = 299 -FINAL_HEIGHT = 399 - - -class AssumeutxoTest(BitcoinTestFramework): - - def set_test_params(self): - """Use the pregenerated, deterministic chain up to height 199.""" - self.num_nodes = 4 - self.rpc_timeout = 120 - self.extra_args = [ - [], - ["-fastprune", "-prune=1"], - ["-persistmempool=0"], - [] - ] - - def setup_network(self): - """Start with the nodes disconnected so that one can generate a snapshot - including blocks the other hasn't yet seen.""" - self.add_nodes(4) - self.start_nodes(extra_args=self.extra_args) - - def test_invalid_snapshot_scenarios(self, valid_snapshot_path): - self.log.info("Test different scenarios of loading invalid snapshot files") - with open(valid_snapshot_path, 'rb') as f: - valid_snapshot_contents = f.read() - bad_snapshot_path = valid_snapshot_path + '.mod' - node = self.nodes[1] - - def expected_error(msg): - assert_raises_rpc_error(-32603, f"Unable to load UTXO snapshot: Population failed: {msg}", node.loadtxoutset, bad_snapshot_path) - - self.log.info(" - snapshot file with invalid file magic") - parsing_error_code = -22 - bad_magic = 0xf00f00f000 - with open(bad_snapshot_path, 'wb') as f: - f.write(bad_magic.to_bytes(5, "big") + valid_snapshot_contents[5:]) - assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format.", node.loadtxoutset, bad_snapshot_path) - - self.log.info(" - snapshot file with unsupported version") - for version in [0, 1, 3]: - with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:5] + version.to_bytes(2, "little") + valid_snapshot_contents[7:]) - assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: Version of snapshot {version} does not match any of the supported versions.", node.loadtxoutset, bad_snapshot_path) - - self.log.info(" - snapshot file with mismatching network magic") - invalid_magics = [ - # magic, name, real - [MAGIC_BYTES["mainnet"], "main", True], - [MAGIC_BYTES["testnet4"], "testnet4", True], - [MAGIC_BYTES["signet"], "signet", True], - [0x00000000.to_bytes(4, 'big'), "", False], - [0xffffffff.to_bytes(4, 'big'), "", False], - ] - for [magic, name, real] in invalid_magics: - with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:7] + magic + valid_snapshot_contents[11:]) - if real: - assert_raises_rpc_error(parsing_error_code, f"Unable to parse metadata: The network of the snapshot ({name}) does not match the network of this node (regtest).", node.loadtxoutset, bad_snapshot_path) - else: - assert_raises_rpc_error(parsing_error_code, "Unable to parse metadata: This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption.", node.loadtxoutset, bad_snapshot_path) - - self.log.info(" - snapshot file referring to a block that is not in the assumeutxo parameters") - prev_block_hash = self.nodes[0].getblockhash(SNAPSHOT_BASE_HEIGHT - 1) - bogus_block_hash = "0" * 64 # Represents any unknown block hash - for bad_block_hash in [bogus_block_hash, prev_block_hash]: - with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:11] + bytes.fromhex(bad_block_hash)[::-1] + valid_snapshot_contents[43:]) - - msg = f"Unable to load UTXO snapshot: assumeutxo block hash in snapshot metadata not recognized (hash: {bad_block_hash}). The following snapshot heights are available: 110, 200, 299." - assert_raises_rpc_error(-32603, msg, node.loadtxoutset, bad_snapshot_path) - - self.log.info(" - snapshot file with wrong number of coins") - valid_num_coins = int.from_bytes(valid_snapshot_contents[43:43 + 8], "little") - for off in [-1, +1]: - with open(bad_snapshot_path, 'wb') as f: - f.write(valid_snapshot_contents[:43]) - f.write((valid_num_coins + off).to_bytes(8, "little")) - f.write(valid_snapshot_contents[43 + 8:]) - expected_error(msg="Bad snapshot - coins left over after deserializing 298 coins." if off == -1 else "Bad snapshot format or truncated snapshot after deserializing 299 coins.") - - self.log.info(" - snapshot file with alternated but parsable UTXO data results in different hash") - cases = [ - # (content, offset, wrong_hash, custom_message) - [b"\xff" * 32, 0, "77874d48d932a5cb7a7f770696f5224ff05746fdcf732a58270b45da0f665934", None], # wrong outpoint hash - [(2).to_bytes(1, "little"), 32, None, "Bad snapshot format or truncated snapshot after deserializing 1 coins."], # wrong txid coins count - [b"\xfd\xff\xff", 32, None, "Mismatch in coins count in snapshot metadata and actual snapshot data"], # txid coins count exceeds coins left - [b"\x01", 33, "9f562925721e4f97e6fde5b590dbfede51e2204a68639525062ad064545dd0ea", None], # wrong outpoint index - [b"\x82", 34, "161393f07f8ad71760b3910a914f677f2cb166e5bcf5354e50d46b78c0422d15", None], # wrong coin code VARINT - [b"\x80", 34, "e6fae191ef851554467b68acff01ca09ad0a2e48c9b3dfea46cf7d35a7fd0ad0", None], # another wrong coin code - [b"\x84\x58", 34, None, "Bad snapshot data after deserializing 0 coins"], # wrong coin case with height 364 and coinbase 0 - [ - # compressed txout value + scriptpubkey - ser_varint(compress_amount(MAX_MONEY + 1)) + ser_varint(0), - # txid + coins per txid + vout + coin height - 32 + 1 + 1 + 2, - None, - "Bad snapshot data after deserializing 0 coins - bad tx out value" - ], # Amount exceeds MAX_MONEY - ] - - for content, offset, wrong_hash, custom_message in cases: - with open(bad_snapshot_path, "wb") as f: - # Prior to offset: Snapshot magic, snapshot version, network magic, hash, coins count - f.write(valid_snapshot_contents[:(5 + 2 + 4 + 32 + 8 + offset)]) - f.write(content) - f.write(valid_snapshot_contents[(5 + 2 + 4 + 32 + 8 + offset + len(content)):]) - - msg = custom_message if custom_message is not None else f"Bad snapshot content hash: expected d2b051ff5e8eef46520350776f4100dd710a63447a8e01d917e92e79751a63e2, got {wrong_hash}." - expected_error(msg) - - def test_headers_not_synced(self, valid_snapshot_path): - for node in self.nodes[1:]: - msg = "Unable to load UTXO snapshot: The base block header (7cc695046fec709f8c9394b6f928f81e81fd3ac20977bb68760fa1faa7916ea2) must appear in the headers chain. Make sure all headers are syncing, and call loadtxoutset again." - assert_raises_rpc_error(-32603, msg, node.loadtxoutset, valid_snapshot_path) - - def test_invalid_chainstate_scenarios(self): - self.log.info("Test different scenarios of invalid snapshot chainstate in datadir") - - self.log.info(" - snapshot chainstate referring to a block that is not in the assumeutxo parameters") - self.stop_node(0) - chainstate_snapshot_path = self.nodes[0].chain_path / "chainstate_snapshot" - chainstate_snapshot_path.mkdir() - with open(chainstate_snapshot_path / "base_blockhash", 'wb') as f: - f.write(b'z' * 32) - - def expected_error(log_msg="", error_msg=""): - with self.nodes[0].assert_debug_log([log_msg]): - self.nodes[0].assert_start_raises_init_error(expected_msg=error_msg) - - expected_error_msg = "Error: A fatal internal error occurred, see debug.log for details: Assumeutxo data not found for the given blockhash '7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a7a'." - error_details = "Assumeutxo data not found for the given blockhash" - expected_error(log_msg=error_details, error_msg=expected_error_msg) - - # resurrect node again - (chainstate_snapshot_path / "base_blockhash").unlink() - chainstate_snapshot_path.rmdir() - self.start_node(0) - - def test_invalid_mempool_state(self, dump_output_path): - self.log.info("Test bitcoind should fail when mempool not empty.") - node=self.nodes[2] - tx = MiniWallet(node).send_self_transfer(from_node=node) - - assert tx['txid'] in node.getrawmempool() - - # Attempt to load the snapshot on Node 2 and expect it to fail - msg = "Unable to load UTXO snapshot: Can't activate a snapshot when mempool not empty" - assert_raises_rpc_error(-32603, msg, node.loadtxoutset, dump_output_path) - - self.restart_node(2, extra_args=self.extra_args[2]) - - def test_invalid_file_path(self): - self.log.info("Test bitcoind should fail when file path is invalid.") - node = self.nodes[0] - path = node.datadir_path / node.chain / "invalid" / "path" - assert_raises_rpc_error(-8, "Couldn't open file {} for reading.".format(path), node.loadtxoutset, path) - - def test_snapshot_with_less_work(self, dump_output_path): - self.log.info("Test bitcoind should fail when snapshot has less accumulated work than this node.") - node = self.nodes[0] - msg = "Unable to load UTXO snapshot: Population failed: Work does not exceed active chainstate." - assert_raises_rpc_error(-32603, msg, node.loadtxoutset, dump_output_path) - - def test_snapshot_block_invalidated(self, dump_output_path): - self.log.info("Test snapshot is not loaded when base block is invalid.") - node = self.nodes[0] - # We are testing the case where the base block is invalidated itself - # and also the case where one of its parents is invalidated. - for height in [SNAPSHOT_BASE_HEIGHT, SNAPSHOT_BASE_HEIGHT - 1]: - block_hash = node.getblockhash(height) - node.invalidateblock(block_hash) - assert_equal(node.getblockcount(), height - 1) - msg = "Unable to load UTXO snapshot: The base block header (7cc695046fec709f8c9394b6f928f81e81fd3ac20977bb68760fa1faa7916ea2) is part of an invalid chain." - assert_raises_rpc_error(-32603, msg, node.loadtxoutset, dump_output_path) - node.reconsiderblock(block_hash) - - def test_snapshot_in_a_divergent_chain(self, dump_output_path): - n0 = self.nodes[0] - n3 = self.nodes[3] - assert_equal(n0.getblockcount(), FINAL_HEIGHT) - assert_equal(n3.getblockcount(), START_HEIGHT) - - self.log.info("Check importing a snapshot where current chain-tip is not an ancestor of the snapshot block but has less work") - # Generate a divergent chain in n3 up to 298 - self.generate(n3, nblocks=99, sync_fun=self.no_op) - assert_equal(n3.getblockcount(), SNAPSHOT_BASE_HEIGHT - 1) - - # Try importing the snapshot and assert its success - loaded = n3.loadtxoutset(dump_output_path) - assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) - normal, snapshot = n3.getchainstates()["chainstates"] - assert_equal(normal['blocks'], START_HEIGHT + 99) - assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT) - - # Both states should have the same nBits and target - assert_equal(normal['bits'], nbits_str(REGTEST_N_BITS)) - assert_equal(normal['bits'], snapshot['bits']) - assert_equal(normal['target'], target_str(REGTEST_TARGET)) - assert_equal(normal['target'], snapshot['target']) - - # Now lets sync the nodes and wait for the background validation to finish - self.connect_nodes(0, 3) - self.sync_blocks(nodes=(n0, n3)) - self.wait_until(lambda: len(n3.getchainstates()['chainstates']) == 1) - - def test_snapshot_not_on_most_work_chain(self, dump_output_path): - self.log.info("Test snapshot is not loaded when the node knows the headers of another chain with more work.") - node0 = self.nodes[0] - node1 = self.nodes[1] - # Create an alternative chain of 2 new blocks, forking off the main chain at the block before the snapshot block. - # This simulates a longer chain than the main chain when submitting these two block headers to node 1 because it is only aware of - # the main chain headers up to the snapshot height. - parent_block_hash = node0.getblockhash(SNAPSHOT_BASE_HEIGHT - 1) - block_time = node0.getblock(node0.getbestblockhash())['time'] + 1 - fork_block1 = create_block(int(parent_block_hash, 16), height=SNAPSHOT_BASE_HEIGHT, ntime=block_time) - fork_block1.solve() - fork_block2 = create_block(fork_block1.hash_int, height=SNAPSHOT_BASE_HEIGHT + 1, ntime=block_time + 1) - fork_block2.solve() - node1.submitheader(fork_block1.serialize().hex()) - node1.submitheader(fork_block2.serialize().hex()) - msg = "A forked headers-chain with more work than the chain with the snapshot base block header exists. Please proceed to sync without AssumeUtxo." - assert_raises_rpc_error(-32603, msg, node1.loadtxoutset, dump_output_path) - # Cleanup: submit two more headers of the snapshot chain to node 1, so that it is the most-work chain again and loading - # the snapshot in future subtests succeeds - main_block1 = node0.getblock(node0.getblockhash(SNAPSHOT_BASE_HEIGHT + 1), 0) - main_block2 = node0.getblock(node0.getblockhash(SNAPSHOT_BASE_HEIGHT + 2), 0) - node1.submitheader(main_block1) - node1.submitheader(main_block2) - - def test_sync_from_assumeutxo_node(self, snapshot): - """ - This test verifies that: - 1. An IBD node can sync headers from an AssumeUTXO node at any time. - 2. IBD nodes do not request historical blocks from AssumeUTXO nodes while they are syncing the background-chain. - 3. The assumeUTXO node dynamically adjusts the network services it offers according to its state. - 4. IBD nodes can fully sync from AssumeUTXO nodes after they finish the background-chain sync. - """ - self.log.info("Testing IBD-sync from assumeUTXO node") - # Node2 starts clean and loads the snapshot. - # Node3 starts clean and seeks to sync-up from snapshot_node. - miner = self.nodes[0] - snapshot_node = self.nodes[2] - ibd_node = self.nodes[3] - - # Start test fresh by cleaning up node directories - for node in (snapshot_node, ibd_node): - self.stop_node(node.index) - self.cleanup_folder(node.chain_path) - self.start_node(node.index, extra_args=self.extra_args[node.index]) - - # Sync-up headers chain on snapshot_node to load snapshot - headers_provider_conn = snapshot_node.add_p2p_connection(P2PInterface()) - headers_provider_conn.wait_for_getheaders() - msg = msg_headers() - for block_num in range(1, miner.getblockcount()+1): - msg.headers.append(from_hex(CBlockHeader(), miner.getblockheader(miner.getblockhash(block_num), verbose=False))) - headers_provider_conn.send_without_ping(msg) - - # Ensure headers arrived - default_value = {'status': ''} # No status - headers_tip_hash = miner.getbestblockhash() - self.wait_until(lambda: next(filter(lambda x: x['hash'] == headers_tip_hash, snapshot_node.getchaintips()), default_value)['status'] == "headers-only") - snapshot_node.disconnect_p2ps() - - # Load snapshot - snapshot_node.loadtxoutset(snapshot['path']) - - # Connect nodes and verify the ibd_node can sync-up the headers-chain from the snapshot_node - self.connect_nodes(ibd_node.index, snapshot_node.index) - snapshot_block_hash = snapshot['base_hash'] - self.wait_until(lambda: next(filter(lambda x: x['hash'] == snapshot_block_hash, ibd_node.getchaintips()), default_value)['status'] == "headers-only") - - # Once the headers-chain is synced, the ibd_node must avoid requesting historical blocks from the snapshot_node. - # If it does request such blocks, the snapshot_node will ignore requests it cannot fulfill, causing the ibd_node - # to stall. This stall could last for up to 10 min, ultimately resulting in an abrupt disconnection due to the - # ibd_node's perceived unresponsiveness. - ensure_for(duration=3, f=lambda: len(ibd_node.getpeerinfo()[0]['inflight']) == 0) - - # Now disconnect nodes and finish background chain sync - self.disconnect_nodes(ibd_node.index, snapshot_node.index) - self.connect_nodes(snapshot_node.index, miner.index) - self.sync_blocks(nodes=(miner, snapshot_node)) - # Check the base snapshot block was stored and ensure node signals full-node service support - self.wait_until(lambda: not try_rpc(-1, "Block not available (not fully downloaded)", snapshot_node.getblock, snapshot_block_hash)) - self.wait_until(lambda: 'NETWORK' in snapshot_node.getnetworkinfo()['localservicesnames']) - - # Now that the snapshot_node is synced, verify the ibd_node can sync from it - self.connect_nodes(snapshot_node.index, ibd_node.index) - assert 'NETWORK' in ibd_node.getpeerinfo()[0]['servicesnames'] - self.sync_blocks(nodes=(ibd_node, snapshot_node)) - - def test_sync_to_most_work_chain_after_background_validation(self): - """ - After background validation completes, node should be able - to download and process blocks from peers without the snapshot block in their chain. - """ - self.log.info("Testing sync to the most-work chain without the snapshot block after background validation") - - forking_node = self.nodes[0] - snapshot_node = self.nodes[2] # Has already completed background validation - - self.log.info("Forking node switches to an alternative chain that forks one block before the snapshot block") - fork_point = SNAPSHOT_BASE_HEIGHT - 1 - forking_node_old_height = forking_node.getblockcount() - forking_node_old_chainwork = int(forking_node.getblockchaininfo()['chainwork'], 16) - forking_node.invalidateblock(forking_node.getblockhash(fork_point + 1)) - - self.log.info("Mine one more block than original chain to make the new chain have most work") - self.generate(forking_node, nblocks=(forking_node_old_height - fork_point) + 1, sync_fun=self.no_op) - assert int(forking_node.getblockchaininfo()['chainwork'], 16) > forking_node_old_chainwork - - self.log.info("Snapshot node should reorg to the most-work chain without the snapshot block") - self.sync_blocks(nodes=(snapshot_node, forking_node)) - - def assert_only_network_limited_service(self, node): - node_services = node.getnetworkinfo()['localservicesnames'] - assert 'NETWORK' not in node_services - assert 'NETWORK_LIMITED' in node_services - - @contextlib.contextmanager - def assert_disk_cleanup(self, node, assumeutxo_used): - """ - Ensure an assumeutxo node is cleaning up the background chainstate - """ - msg = [] - if assumeutxo_used: - # Check that the snapshot actually existed before restart - assert (node.datadir_path / node.chain / "chainstate_snapshot").exists() - msg = ["cleaning up unneeded background chainstate"] - - with node.assert_debug_log(msg): - yield - - assert not (node.datadir_path / node.chain / "chainstate_snapshot").exists() - - def run_test(self): - """ - Bring up two (disconnected) nodes, mine some new blocks on the first, - and generate a UTXO snapshot. - - Load the snapshot into the second, ensure it syncs to tip and completes - background validation when connected to the first. - """ - n0 = self.nodes[0] - n1 = self.nodes[1] - n2 = self.nodes[2] - n3 = self.nodes[3] - - self.mini_wallet = MiniWallet(n0) - - # Mock time for a deterministic chain - for n in self.nodes: - n.setmocktime(n.getblockheader(n.getbestblockhash())['time']) - - # Generate a series of blocks that `n0` will have in the snapshot, - # but that n1 and n2 don't yet see. - assert_equal(n0.getblockcount(), START_HEIGHT) - blocks = {START_HEIGHT: Block(n0.getbestblockhash(), 1, START_HEIGHT + 1)} - for i in range(100): - block_tx = 1 - if i % 3 == 0: - self.mini_wallet.send_self_transfer(from_node=n0) - block_tx += 1 - self.generate(n0, nblocks=1, sync_fun=self.no_op) - height = n0.getblockcount() - hash = n0.getbestblockhash() - blocks[height] = Block(hash, block_tx, blocks[height-1].chain_tx + block_tx) - if i == 4: - # Create a stale block that forks off the main chain before the snapshot. - temp_invalid = n0.getbestblockhash() - n0.invalidateblock(temp_invalid) - stale_hash = self.generateblock(n0, output="raw(aaaa)", transactions=[], sync_fun=self.no_op)["hash"] - n0.invalidateblock(stale_hash) - n0.reconsiderblock(temp_invalid) - stale_block = n0.getblock(stale_hash, 0) - - - self.log.info("-- Testing assumeutxo + some indexes + pruning") - - assert_equal(n0.getblockcount(), SNAPSHOT_BASE_HEIGHT) - assert_equal(n1.getblockcount(), START_HEIGHT) - - self.log.info(f"Creating a UTXO snapshot at height {SNAPSHOT_BASE_HEIGHT}") - dump_output = n0.dumptxoutset('utxos.dat', "latest") - - self.log.info("Test loading snapshot when the node tip is on the same block as the snapshot") - assert_equal(n0.getblockcount(), SNAPSHOT_BASE_HEIGHT) - assert_equal(n0.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) - self.test_snapshot_with_less_work(dump_output['path']) - - self.log.info("Test loading snapshot when headers are not synced") - self.test_headers_not_synced(dump_output['path']) - - # In order for the snapshot to activate, we have to ferry over the new - # headers to n1 and n2 so that they see the header of the snapshot's - # base block while disconnected from n0. - for i in range(1, 300): - block = n0.getblock(n0.getblockhash(i), 0) - # make n1 and n2 aware of the new header, but don't give them the - # block. - n1.submitheader(block) - n2.submitheader(block) - n3.submitheader(block) - - # Ensure everyone is seeing the same headers. - for n in self.nodes: - assert_equal(n.getblockchaininfo()["headers"], SNAPSHOT_BASE_HEIGHT) - - assert_equal(n0.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) - - def check_dump_output(output): - assert_equal( - output['txoutset_hash'], - "d2b051ff5e8eef46520350776f4100dd710a63447a8e01d917e92e79751a63e2") - assert_equal(output["nchaintx"], blocks[SNAPSHOT_BASE_HEIGHT].chain_tx) - - check_dump_output(dump_output) - - # Mine more blocks on top of the snapshot that n1 hasn't yet seen. This - # will allow us to test n1's sync-to-tip on top of a snapshot. - self.generate(n0, nblocks=100, sync_fun=self.no_op) - - assert_equal(n0.getblockcount(), FINAL_HEIGHT) - assert_equal(n1.getblockcount(), START_HEIGHT) - - assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) - - self.log.info("Check that dumptxoutset works for past block heights") - # rollback defaults to the snapshot base height - dump_output2 = n0.dumptxoutset('utxos2.dat', "rollback") - check_dump_output(dump_output2) - assert_equal(sha256sum_file(dump_output['path']), sha256sum_file(dump_output2['path'])) - - # Rollback with specific height - dump_output3 = n0.dumptxoutset('utxos3.dat', rollback=SNAPSHOT_BASE_HEIGHT) - check_dump_output(dump_output3) - assert_equal(sha256sum_file(dump_output['path']), sha256sum_file(dump_output3['path'])) - - # Specified height that is not a snapshot height - prev_snap_height = SNAPSHOT_BASE_HEIGHT - 1 - dump_output4 = n0.dumptxoutset(path='utxos4.dat', rollback=prev_snap_height) - assert_equal( - dump_output4['txoutset_hash'], - "45ac2777b6ca96588210e2a4f14b602b41ec37b8b9370673048cc0af434a1ec8") - assert_not_equal(sha256sum_file(dump_output['path']), sha256sum_file(dump_output4['path'])) - - # Use a hash instead of a height - prev_snap_hash = n0.getblockhash(prev_snap_height) - dump_output5 = n0.dumptxoutset('utxos5.dat', rollback=prev_snap_hash) - assert_equal(sha256sum_file(dump_output4['path']), sha256sum_file(dump_output5['path'])) - - # Ensure n0 is back at the tip - assert_equal(n0.getblockchaininfo()["blocks"], FINAL_HEIGHT) - - self.test_snapshot_with_less_work(dump_output['path']) - self.test_invalid_mempool_state(dump_output['path']) - self.test_invalid_snapshot_scenarios(dump_output['path']) - self.test_invalid_chainstate_scenarios() - self.test_invalid_file_path() - self.test_snapshot_block_invalidated(dump_output['path']) - self.test_snapshot_not_on_most_work_chain(dump_output['path']) - - # Prune-node sanity check - assert 'NETWORK' not in n1.getnetworkinfo()['localservicesnames'] - - self.log.info(f"Loading snapshot into second node from {dump_output['path']}") - # This node's tip is on an ancestor block of the snapshot, which should - # be the normal case - loaded = n1.loadtxoutset(dump_output['path']) - assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) - assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) - - self.log.info("Confirm that local services remain unchanged") - # Since n1 is a pruned node, the 'NETWORK' service flag must always be unset. - self.assert_only_network_limited_service(n1) - - self.log.info("Check that UTXO-querying RPCs operate on snapshot chainstate") - snapshot_hash = loaded['tip_hash'] - snapshot_num_coins = loaded['coins_loaded'] - utxo_info = n1.gettxoutsetinfo() - assert_equal(utxo_info['txouts'], snapshot_num_coins) - assert_equal(utxo_info['height'], SNAPSHOT_BASE_HEIGHT) - assert_equal(utxo_info['bestblock'], snapshot_hash) - - self.log.info("Check that getblockchaininfo returns information about the background validation process") - expected_keys = [ - "snapshotheight", - "blocks", - "bestblockhash", - "mediantime", - "chainwork", - "verificationprogress" - ] - res = n1.getblockchaininfo() - assert "backgroundvalidation" in res.keys() - bv_res = res["backgroundvalidation"] - assert_equal(sorted(expected_keys), sorted(bv_res.keys())) - assert_equal(bv_res["snapshotheight"], SNAPSHOT_BASE_HEIGHT) - assert_equal(bv_res["blocks"], START_HEIGHT) - assert_equal(bv_res["bestblockhash"], n1.getblockhash(START_HEIGHT)) - block = n1.getblockheader(bv_res["bestblockhash"]) - assert_equal(bv_res["mediantime"], block["mediantime"]) - assert_equal(bv_res["chainwork"], block["chainwork"]) - background_tx_count = n1.getchaintxstats(blockhash=bv_res["bestblockhash"])["txcount"] - snapshot_tx_count = n1.getchaintxstats(blockhash=snapshot_hash)["txcount"] - expected_verification_progress = background_tx_count / snapshot_tx_count - assert_approx(bv_res["verificationprogress"], expected_verification_progress, vspan=0.01) - - # find coinbase output at snapshot height on node0 and scan for it on node1, - # where the block is not available, but the snapshot was loaded successfully - coinbase_tx = n0.getblock(snapshot_hash, verbosity=2)['tx'][0] - assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", n1.getblock, snapshot_hash) - coinbase_output_descriptor = coinbase_tx['vout'][0]['scriptPubKey']['desc'] - scan_result = n1.scantxoutset('start', [coinbase_output_descriptor]) - assert_equal(scan_result['success'], True) - assert_equal(scan_result['txouts'], snapshot_num_coins) - assert_equal(scan_result['height'], SNAPSHOT_BASE_HEIGHT) - assert_equal(scan_result['bestblock'], snapshot_hash) - scan_utxos = [(coin['txid'], coin['vout']) for coin in scan_result['unspents']] - assert (coinbase_tx['txid'], 0) in scan_utxos - - txout_result = n1.gettxout(coinbase_tx['txid'], 0) - assert_equal(txout_result['scriptPubKey']['desc'], coinbase_output_descriptor) - - def check_tx_counts(final: bool) -> None: - """Check nTx and nChainTx intermediate values right after loading - the snapshot, and final values after the snapshot is validated.""" - for height, block in blocks.items(): - tx = n1.getblockheader(block.hash)["nTx"] - stats = n1.getchaintxstats(nblocks=1, blockhash=block.hash) - chain_tx = stats.get("txcount", None) - window_tx_count = stats.get("window_tx_count", None) - tx_rate = stats.get("txrate", None) - window_interval = stats.get("window_interval") - - # Intermediate nTx of the starting block should be set, but nTx of - # later blocks should be 0 before they are downloaded. - # The window_tx_count of one block is equal to the blocks tx count. - # If the window tx count is unknown, the value is missing. - # The tx_rate is calculated from window_tx_count and window_interval - # when possible. - if final or height == START_HEIGHT: - assert_equal(tx, block.tx) - assert_equal(window_tx_count, tx) - if window_interval > 0: - assert_approx(tx_rate, window_tx_count / window_interval, vspan=0.1) - else: - assert_equal(tx_rate, None) - else: - assert_equal(tx, 0) - assert_equal(window_tx_count, None) - - # Intermediate nChainTx of the starting block and snapshot block - # should be set, but others should be None until they are downloaded. - if final or height in (START_HEIGHT, SNAPSHOT_BASE_HEIGHT): - assert_equal(chain_tx, block.chain_tx) - else: - assert_equal(chain_tx, None) - - check_tx_counts(final=False) - - normal, snapshot = n1.getchainstates()["chainstates"] - assert_equal(normal['blocks'], START_HEIGHT) - assert_equal(normal.get('snapshot_blockhash'), None) - assert_equal(normal['validated'], True) - assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT) - assert_equal(snapshot['snapshot_blockhash'], dump_output['base_hash']) - assert_equal(snapshot['validated'], False) - - assert_equal(n1.getblockchaininfo()["blocks"], SNAPSHOT_BASE_HEIGHT) - - self.log.info("Submit a stale block that forked off the chain before the snapshot") - # Normally a block like this would not be downloaded, but if it is - # submitted early before the background chain catches up to the fork - # point, it winds up in m_blocks_unlinked and triggers a corner case - # that previously crashed CheckBlockIndex. - n1.submitblock(stale_block) - n1.getchaintips() - n1.getblock(stale_hash) - - self.log.info("Submit a spending transaction for a snapshot chainstate coin to the mempool") - # spend the coinbase output of the first block that is not available on node1 - spend_coin_blockhash = n1.getblockhash(START_HEIGHT + 1) - assert_raises_rpc_error(-1, "Block not available (not fully downloaded)", n1.getblock, spend_coin_blockhash) - prev_tx = n0.getblock(spend_coin_blockhash, 3)['tx'][0] - prevout = {"txid": prev_tx['txid'], "vout": 0, "scriptPubKey": prev_tx['vout'][0]['scriptPubKey']['hex']} - privkey = n0.get_deterministic_priv_key().key - raw_tx = n1.createrawtransaction([prevout], {getnewdestination()[2]: 24.99}) - signed_tx = n1.signrawtransactionwithkey(raw_tx, [privkey], [prevout])['hex'] - signed_txid = tx_from_hex(signed_tx).txid_hex - - assert n1.gettxout(prev_tx['txid'], 0) is not None - n1.sendrawtransaction(signed_tx) - assert signed_txid in n1.getrawmempool() - assert not n1.gettxout(prev_tx['txid'], 0) - - PAUSE_HEIGHT = FINAL_HEIGHT - 40 - - self.log.info(f"Sync node up to height {PAUSE_HEIGHT}") - # During snapshot tip sync, the node must remain in 'limited' mode. - self.assert_only_network_limited_service(n1) - dumb_sync_blocks(src=n0, dst=n1, height=PAUSE_HEIGHT) - - self.log.info("Checking that blocks are segmented on disk") - assert self.has_blockfile(n1, "00000"), "normal blockfile missing" - assert self.has_blockfile(n1, "00001"), "assumed blockfile missing" - assert not self.has_blockfile(n1, "00002"), "too many blockfiles" - - # The node must remain in 'limited' mode - self.assert_only_network_limited_service(n1) - - # Send snapshot block to n1 out of order. This makes the test less - # realistic because normally the snapshot block is one of the last - # blocks downloaded, but its useful to test because it triggers more - # corner cases in ReceivedBlockTransactions() and CheckBlockIndex() - # setting and testing nChainTx values, and it exposed previous bugs. - snapshot_hash = n0.getblockhash(SNAPSHOT_BASE_HEIGHT) - snapshot_block = n0.getblock(snapshot_hash, 0) - n1.submitblock(snapshot_block) - - self.connect_nodes(0, 1) - - self.log.info(f"Ensuring snapshot chain syncs to tip. ({FINAL_HEIGHT})") - self.wait_until(lambda: n1.getchainstates()['chainstates'][-1]['blocks'] == FINAL_HEIGHT) - self.sync_blocks(nodes=(n0, n1)) - - self.log.info("Ensuring background validation completes") - self.wait_until(lambda: len(n1.getchainstates()['chainstates']) == 1) - - # Since n1 is a pruned node, it will not signal NODE_NETWORK after - # completing the background sync. - self.assert_only_network_limited_service(n1) - - self.log.info("Re-check nTx and nChainTx values") - check_tx_counts(final=True) - - for i in (0, 1): - n = self.nodes[i] - self.log.info(f"Restarting node {i} to ensure (Check|Load)BlockIndex passes") - with self.assert_disk_cleanup(n, i == 1): - self.restart_node(i, extra_args=self.extra_args[i]) - - assert_equal(n.getblockchaininfo()["blocks"], FINAL_HEIGHT) - - chainstate, = n.getchainstates()['chainstates'] - assert_equal(chainstate['blocks'], FINAL_HEIGHT) - - - # Node 2: reindex - # --------------- - - self.log.info("-- Testing reindex") - assert_equal(n2.getblockcount(), START_HEIGHT) - assert 'NETWORK' in n2.getnetworkinfo()['localservicesnames'] # sanity check - - self.log.info(f"Loading snapshot into third node from {dump_output['path']}") - loaded = n2.loadtxoutset(dump_output['path']) - assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) - assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) - - # Even though n2 is a full node, it will unset the 'NETWORK' service flag during snapshot loading. - # This indicates other peers that the node will temporarily not provide historical blocks. - self.log.info("Check node2 updated the local services during snapshot load") - self.assert_only_network_limited_service(n2) - - for reindex_arg in ['-reindex=1', '-reindex-chainstate=1']: - self.log.info(f"Check that restarting with {reindex_arg} will delete the snapshot chainstate") - self.restart_node(2, extra_args=[reindex_arg, *self.extra_args[2]]) - assert_equal(1, len(n2.getchainstates()["chainstates"])) - for i in range(1, 300): - block = n0.getblock(n0.getblockhash(i), 0) - n2.submitheader(block) - loaded = n2.loadtxoutset(dump_output['path']) - assert_equal(loaded['coins_loaded'], SNAPSHOT_BASE_HEIGHT) - assert_equal(loaded['base_height'], SNAPSHOT_BASE_HEIGHT) - - normal, snapshot = n2.getchainstates()['chainstates'] - assert_equal(normal['blocks'], START_HEIGHT) - assert_equal(normal.get('snapshot_blockhash'), None) - assert_equal(normal['validated'], True) - assert_equal(snapshot['blocks'], SNAPSHOT_BASE_HEIGHT) - assert_equal(snapshot['snapshot_blockhash'], dump_output['base_hash']) - assert_equal(snapshot['validated'], False) - - self.log.info("Check that loading the snapshot again will fail because there is already an active snapshot.") - msg = "Unable to load UTXO snapshot: Can't activate a snapshot-based chainstate more than once" - assert_raises_rpc_error(-32603, msg, n2.loadtxoutset, dump_output['path']) - - # Upon restart, the node must stay in 'limited' mode until the background - # chain sync completes. - self.restart_node(2, extra_args=self.extra_args[2]) - self.assert_only_network_limited_service(n2) - - self.connect_nodes(0, 2) - self.wait_until(lambda: n2.getchainstates()['chainstates'][-1]['blocks'] == FINAL_HEIGHT) - self.sync_blocks(nodes=(n0, n2)) - - self.log.info("Ensuring background validation completes") - self.wait_until(lambda: len(n2.getchainstates()['chainstates']) == 1) - - # Once background chain sync completes, the full node must start offering historical blocks again. - self.wait_until(lambda: {'NETWORK', 'NETWORK_LIMITED'}.issubset(n2.getnetworkinfo()['localservicesnames'])) - - for i in (0, 2): - n = self.nodes[i] - self.log.info(f"Restarting node {i} to ensure (Check|Load)BlockIndex passes") - with self.assert_disk_cleanup(n, i == 2): - self.restart_node(i, extra_args=self.extra_args[i]) - - assert_equal(n.getblockchaininfo()["blocks"], FINAL_HEIGHT) - - chainstate, = n.getchainstates()['chainstates'] - assert_equal(chainstate['blocks'], FINAL_HEIGHT) - - self.log.info("Test -reindex-chainstate of an assumeutxo-synced node") - self.restart_node(2, extra_args=[ - '-reindex-chainstate=1', *self.extra_args[2]]) - assert_equal(n2.getblockchaininfo()["blocks"], FINAL_HEIGHT) - self.wait_until(lambda: n2.getblockcount() == FINAL_HEIGHT) - - self.log.info("Test -reindex of an assumeutxo-synced node") - self.restart_node(2, extra_args=['-reindex=1', *self.extra_args[2]]) - self.connect_nodes(0, 2) - self.wait_until(lambda: n2.getblockcount() == FINAL_HEIGHT) - - self.test_snapshot_in_a_divergent_chain(dump_output['path']) - - # The following test cleans node2 and node3 chain directories. - self.test_sync_from_assumeutxo_node(snapshot=dump_output) - - self.test_sync_to_most_work_chain_after_background_validation() - -@dataclass -class Block: - hash: str - tx: int - chain_tx: int - -if __name__ == '__main__': - AssumeutxoTest(__file__).main() diff --git a/test/functional/tool_bitcoin_chainstate.py b/test/functional/tool_bitcoin_chainstate.py deleted file mode 100755 index c57da1fc376d..000000000000 --- a/test/functional/tool_bitcoin_chainstate.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2022-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -"""Test bitcoin-chainstate tool functionality - -Test basic block processing via bitcoin-chainstate tool, including detecting -duplicates and malformed input. - -Test that bitcoin-chainstate can load a datadir initialized with an assumeutxo -snapshot and extend the snapshot chain with new blocks. -""" - -import subprocess - -from test_framework.test_framework import BitcoinTestFramework -from test_framework.util import assert_equal -from test_framework.wallet import MiniWallet - -START_HEIGHT = 199 -# Hardcoded in regtest chainparams -SNAPSHOT_BASE_BLOCK_HEIGHT = 299 -SNAPSHOT_BASE_BLOCK_HASH = "7cc695046fec709f8c9394b6f928f81e81fd3ac20977bb68760fa1faa7916ea2" - - -class BitcoinChainstateTest(BitcoinTestFramework): - def skip_test_if_missing_module(self): - self.skip_if_no_bitcoin_chainstate() - - def set_test_params(self): - """Use the pregenerated, deterministic chain up to height 199.""" - self.num_nodes = 2 - - def setup_network(self): - """Start with the nodes disconnected so that one can generate a snapshot - including blocks the other hasn't yet seen.""" - self.add_nodes(2) - self.start_nodes() - - def generate_snapshot_chain(self): - self.log.info(f"Generate deterministic chain up to block {SNAPSHOT_BASE_BLOCK_HEIGHT} for node0 while node1 disconnected") - n0 = self.nodes[0] - assert_equal(n0.getblockcount(), START_HEIGHT) - n0.setmocktime(n0.getblockheader(n0.getbestblockhash())['time']) - mini_wallet = MiniWallet(n0) - for i in range(SNAPSHOT_BASE_BLOCK_HEIGHT - n0.getblockchaininfo()["blocks"]): - if i % 3 == 0: - mini_wallet.send_self_transfer(from_node=n0) - self.generate(n0, nblocks=1, sync_fun=self.no_op) - assert_equal(n0.getblockcount(), SNAPSHOT_BASE_BLOCK_HEIGHT) - assert_equal(n0.getbestblockhash(), SNAPSHOT_BASE_BLOCK_HASH) - return n0.dumptxoutset('utxos.dat', "latest") - - def add_block(self, datadir, input, *, expected_stderr=None, expected_stdout=None): - proc = subprocess.Popen( - self.get_binaries().chainstate_argv() + ["-regtest", datadir], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - ) - stdout, stderr = proc.communicate(input=input + "\n", timeout=5 * self.options.timeout_factor) - self.log.debug("STDOUT: {0}".format(stdout.strip("\n"))) - self.log.info("STDERR: {0}".format(stderr.strip("\n"))) - - if expected_stderr is not None and expected_stderr not in stderr: - raise AssertionError(f"Expected stderr output '{expected_stderr}' does not partially match stderr:\n{stderr}") - if expected_stdout is not None and expected_stdout not in stdout: - raise AssertionError(f"Expected stdout output '{expected_stdout}' does not partially match stdout:\n{stdout}") - - def basic_test(self): - n0 = self.nodes[0] - n1 = self.nodes[1] - datadir = n1.chain_path - n1.stop_node() - block = n0.getblock(n0.getblockhash(START_HEIGHT+1), 0) - self.log.info(f"Test bitcoin-chainstate {self.get_binaries().chainstate_argv()} with datadir: {datadir}") - self.add_block(datadir, block, expected_stderr="Block has not yet been rejected") - self.add_block(datadir, block, expected_stderr="duplicate") - self.add_block(datadir, "00", expected_stderr="Block decode failed") - self.add_block(datadir, "", expected_stderr="Empty line found") - - def assumeutxo_test(self, dump_output_path): - n0 = self.nodes[0] - n1 = self.nodes[1] - self.start_node(1) - self.log.info("Submit headers for new blocks to node1, then load the snapshot so it activates") - for height in range(START_HEIGHT+2, SNAPSHOT_BASE_BLOCK_HEIGHT+1): - block = n0.getblock(n0.getblockhash(height), 0) - n1.submitheader(block) - assert_equal(n1.getblockcount(), START_HEIGHT+1) - loaded = n1.loadtxoutset(dump_output_path) - assert_equal(loaded['base_height'], SNAPSHOT_BASE_BLOCK_HEIGHT) - datadir = n1.chain_path - n1.stop_node() - self.log.info(f"Test bitcoin-chainstate {self.get_binaries().chainstate_argv()} with an assumeutxo datadir: {datadir}") - new_tip_hash = self.generate(n0, nblocks=1, sync_fun=self.no_op)[0] - self.add_block(datadir, n0.getblock(new_tip_hash, 0), expected_stdout="Block tip changed") - - def run_test(self): - dump_output = self.generate_snapshot_chain() - self.basic_test() - self.assumeutxo_test(dump_output['path']) - -if __name__ == "__main__": - BitcoinChainstateTest(__file__).main() From 7fac466d286642e8e069ae13c69dfc51aa14e0c7 Mon Sep 17 00:00:00 2001 From: josie Date: Tue, 12 May 2026 21:25:36 +0200 Subject: [PATCH 2/3] nuke: assumeutxo (commit 2 of 3) - rip snapshot machinery Delete AssumeutxoHash/AssumeutxoData/AssumeutxoForHeight/ForBlockhash and GetAvailableSnapshotHeights from chainparams. Delete ChainstateRole, the Assumeutxo enum, SnapshotMetadata, SnapshotCompletionResult, and the m_from_snapshot_blockhash / m_target_blockhash / m_target_utxohash / m_cached_snapshot_base / m_cached_target_block members on Chainstate. Drop ActivateSnapshot, PopulateAndValidateSnapshot, MaybeValidateSnapshot, DetectSnapshotChainstate, ActivateExistingSnapshot, HistoricalChainstate, ValidatedChainstate, RemoveChainstate, LoadAssumeutxoChainstate, ValidatedSnapshotCleanup, GetBackgroundVerificationProgress and the snapshot-aware paths in CheckBlockIndex / LoadBlockIndex / LoadChainTip / UpdateTipLog / MaybeRebalanceCaches. Delete BLOCK_ASSUMED_VALID, BlockManager::m_snapshot_height, the per-snapshot blockfile cursor array, and the NODE_NETWORK-flip on snapshot load. Drop the ChainstateRole parameter from BlockConnected and ChainStateFlushed across validationinterface, node/interfaces, zmqnotificationinterface, bitcoinkernel. Drop Chain::hasAssumedValidChain. ChainstateManager still holds std::vector> m_chainstates with exactly one entry; commit 3 collapses that to a single unique_ptr. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/CMakeLists.txt | 1 - src/chain.h | 22 +- src/coins.h | 4 +- src/init.cpp | 19 +- src/interfaces/chain.h | 12 +- src/kernel/CMakeLists.txt | 1 - src/kernel/bitcoinkernel.cpp | 3 +- src/kernel/chain.cpp | 14 - src/kernel/chain.h | 4 - src/kernel/chainparams.cpp | 11 - src/kernel/chainparams.h | 39 - src/kernel/types.h | 30 - src/net.h | 4 +- src/net_processing.cpp | 67 +- src/node/blockstorage.cpp | 126 +- src/node/blockstorage.h | 52 +- src/node/chainstate.cpp | 74 +- src/node/interfaces.cpp | 15 +- src/node/utxo_snapshot.cpp | 94 -- src/node/utxo_snapshot.h | 136 --- src/rpc/blockchain.cpp | 43 +- src/rpc/client.cpp | 1 - src/rpc/rawtransaction_util.cpp | 1 + src/test/chainstate_write_tests.cpp | 6 +- src/test/fuzz/deserialize.cpp | 7 - src/test/fuzz/rpc.cpp | 1 - src/test/util/chainstate.h | 139 --- src/test/util/validation.cpp | 5 +- src/test/util/validation.h | 1 - src/test/validation_block_tests.cpp | 3 +- src/test/validation_chainstate_tests.cpp | 1 - .../validation_chainstatemanager_tests.cpp | 517 -------- src/validation.cpp | 1062 +---------------- src/validation.h | 259 +--- src/validationinterface.cpp | 15 +- src/validationinterface.h | 14 +- src/zmq/zmqnotificationinterface.cpp | 7 +- src/zmq/zmqnotificationinterface.h | 2 +- 38 files changed, 154 insertions(+), 2658 deletions(-) delete mode 100644 src/kernel/types.h delete mode 100644 src/node/utxo_snapshot.cpp delete mode 100644 src/node/utxo_snapshot.h delete mode 100644 src/test/util/chainstate.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ed4d9d83ddb6..00d3a20b4aff 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -205,7 +205,6 @@ add_library(bitcoin_node STATIC EXCLUDE_FROM_ALL node/txdownloadman_impl.cpp node/txorphanage.cpp node/txreconciliation.cpp - node/utxo_snapshot.cpp node/warnings.cpp noui.cpp policy/ephemeral_policy.cpp diff --git a/src/chain.h b/src/chain.h index 7701e9262a42..01eb1c1f59eb 100644 --- a/src/chain.h +++ b/src/chain.h @@ -55,17 +55,15 @@ enum BlockStatus : uint32_t { * sigops, size, merkle root. Implies all parents are at least TREE but not necessarily TRANSACTIONS. * * If a block's validity is at least VALID_TRANSACTIONS, CBlockIndex::nTx will be set. If a block and all previous - * blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_TRANSACTIONS, - * CBlockIndex::m_chain_tx_count will be set. + * blocks back to the genesis block are at least VALID_TRANSACTIONS, CBlockIndex::m_chain_tx_count will be set. */ BLOCK_VALID_TRANSACTIONS = 3, //! Outputs do not overspend inputs, no double spends, coinbase output ok, no immature coinbase spends, BIP30. - //! Implies all previous blocks back to the genesis block or an assumeutxo snapshot block are at least VALID_CHAIN. + //! Implies all previous blocks back to the genesis block are at least VALID_CHAIN. BLOCK_VALID_CHAIN = 4, - //! Scripts & signatures ok. Implies all previous blocks back to the genesis block or an assumeutxo snapshot block - //! are at least VALID_SCRIPTS. + //! Scripts & signatures ok. Implies all previous blocks back to the genesis block are at least VALID_SCRIPTS. BLOCK_VALID_SCRIPTS = 5, //! All validity bits. @@ -81,8 +79,7 @@ enum BlockStatus : uint32_t { BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client - BLOCK_STATUS_RESERVED = 256, //!< Unused flag that was previously set on assumeutxo snapshot blocks and their - //!< ancestors before they were validated, and unset when they were validated. + BLOCK_STATUS_RESERVED = 256, //!< Unused flag retained for backward compatibility with existing block index entries on disk. }; /** The block chain is a tree shaped structure starting with the @@ -124,16 +121,12 @@ class CBlockIndex //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero if this block and all previous blocks back - //! to the genesis block or an assumeutxo snapshot block have reached the - //! VALID_TRANSACTIONS level. + //! to the genesis block have reached the VALID_TRANSACTIONS level. uint64_t m_chain_tx_count{0}; //! Verification status of this block. See enum BlockStatus //! - //! Note: this value is modified to show BLOCK_OPT_WITNESS during UTXO snapshot - //! load to avoid a spurious startup failure requiring -reindex. //! @sa NeedsRedownload - //! @sa ActivateSnapshot uint32_t nStatus GUARDED_BY(::cs_main){0}; //! block header @@ -202,14 +195,11 @@ class CBlockIndex } /** - * Check whether this block and all previous blocks back to the genesis block or an assumeutxo snapshot block have + * Check whether this block and all previous blocks back to the genesis block have * reached VALID_TRANSACTIONS and had transactions downloaded (and stored to disk) at some point. * * Does not imply the transactions are consensus-valid (ConnectTip might fail) * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true) - * - * Note that this will be true for the snapshot base block, if one is loaded, since its m_chain_tx_count value will have - * been set manually based on the related AssumeutxoData entry. */ bool HaveNumChainTxs() const { return m_chain_tx_count != 0; } diff --git a/src/coins.h b/src/coins.h index a7a53118108d..fd781fb74369 100644 --- a/src/coins.h +++ b/src/coins.h @@ -468,8 +468,8 @@ class CCoinsViewCache : public CCoinsViewBacked * Emplace a coin into cacheCoins without performing any checks, marking * the emplaced coin as dirty. * - * NOT FOR GENERAL USE. Used only when loading coins from a UTXO snapshot. - * @sa ChainstateManager::PopulateAndValidateSnapshot() + * NOT FOR GENERAL USE. Bypasses normal cache-state bookkeeping; intended + * only for bulk loading paths and tests that have to fabricate coin entries. */ void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin); diff --git a/src/init.cpp b/src/init.cpp index d3ebb2c84717..a7fd18fc5572 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -500,8 +500,8 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc) argsman.AddArg("-prune=", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1_MiB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex", "If enabled, wipe chain state and block index, and rebuild them from blk*.dat files on disk. If an assumeutxo snapshot was loaded, its chainstate will be wiped as well. The snapshot can then be reloaded via RPC.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex-chainstate", "If enabled, wipe chain state, and rebuild it from blk*.dat files on disk. If an assumeutxo snapshot was loaded, its chainstate will be wiped as well. The snapshot can then be reloaded via RPC.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex", "If enabled, wipe chain state and block index, and rebuild them from blk*.dat files on disk.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex-chainstate", "If enabled, wipe chain state, and rebuild it from blk*.dat files on disk.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-settings=", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -1298,12 +1298,6 @@ static ChainstateLoadResult InitAndLoadChainstate( ChainstateManager& chainman = *node.chainman; if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; - chainman.snapshot_download_completed = [&node]() { - if (!node.chainman->m_blockman.IsPruneMode()) { - LogInfo("[snapshot] re-enabling NODE_NETWORK services"); - node.connman->AddLocalServices(NODE_NETWORK); - } - }; node::ChainstateLoadOptions options; options.mempool = Assert(node.mempool.get()); options.wipe_chainstate_db = do_reindex || do_reindex_chainstate; @@ -1809,13 +1803,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } } } else { - // Prior to setting NODE_NETWORK, check if we can provide historical blocks. - if (!WITH_LOCK(chainman.GetMutex(), return chainman.HistoricalChainstate())) { - LogInfo("Setting NODE_NETWORK in non-prune mode"); - g_local_services = ServiceFlags(g_local_services | NODE_NETWORK); - } else { - LogInfo("Running node in NODE_NETWORK_LIMITED mode until snapshot background sync completes"); - } + LogInfo("Setting NODE_NETWORK in non-prune mode"); + g_local_services = ServiceFlags(g_local_services | NODE_NETWORK); } // ********************************************************* Step 11: import blocks diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index b04f11890689..ff59b41bf425 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -33,9 +33,6 @@ enum class RBFTransactionState; struct bilingual_str; struct CBlockLocator; struct FeeCalculation; -namespace kernel { -struct ChainstateRole; -} // namespace kernel namespace node { struct NodeContext; } // namespace node @@ -299,10 +296,10 @@ class Chain virtual ~Notifications() = default; virtual void transactionAddedToMempool(const CTransactionRef& tx) {} virtual void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason) {} - virtual void blockConnected(const kernel::ChainstateRole& role, const BlockInfo& block) {} + virtual void blockConnected(const BlockInfo& block) {} virtual void blockDisconnected(const BlockInfo& block) {} virtual void updatedBlockTip() {} - virtual void chainStateFlushed(const kernel::ChainstateRole& role, const CBlockLocator& locator) {} + virtual void chainStateFlushed(const CBlockLocator& locator) {} }; //! Options specifying which chain notifications are required. @@ -373,11 +370,6 @@ class Chain //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; - //! Return true if an assumed-valid snapshot is in use. Note that this - //! returns true even after the snapshot is validated, until the next node - //! restart. - virtual bool hasAssumedValidChain() = 0; - //! Get internal node context. Useful for testing, but not //! accessible across processes. virtual node::NodeContext* context() { return nullptr; } diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 541f10b3adce..5d7db3e6cf8f 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -33,7 +33,6 @@ add_library(bitcoinkernel ../logging.cpp ../node/blockstorage.cpp ../node/chainstate.cpp - ../node/utxo_snapshot.cpp ../policy/ephemeral_policy.cpp ../policy/feerate.cpp ../policy/packages.cpp diff --git a/src/kernel/bitcoinkernel.cpp b/src/kernel/bitcoinkernel.cpp index aa095cab74f9..0b640eb45ab3 100644 --- a/src/kernel/bitcoinkernel.cpp +++ b/src/kernel/bitcoinkernel.cpp @@ -55,7 +55,6 @@ namespace Consensus { struct Params; } // namespace Consensus -using kernel::ChainstateRole; using util::ImmediateTaskRunner; // Define G_TRANSLATION_FUN symbol in libbitcoinkernel library so users of the @@ -365,7 +364,7 @@ class KernelValidationInterface final : public CValidationInterface } } - void BlockConnected(const ChainstateRole& role, const std::shared_ptr& block, const CBlockIndex* pindex) override + void BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex) override { if (m_cbs.block_connected) { m_cbs.block_connected(m_cbs.user_data, diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp index 0a33e9a1f718..828786ac5987 100644 --- a/src/kernel/chain.cpp +++ b/src/kernel/chain.cpp @@ -6,14 +6,11 @@ #include #include -#include #include #include class CBlock; -using kernel::ChainstateRole; - namespace kernel { interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data) { @@ -29,15 +26,4 @@ interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data info.data = data; return info; } - -std::ostream& operator<<(std::ostream& os, const ChainstateRole& role) { - if (!role.validated) { - os << "assumedvalid"; - } else if (role.historical) { - os << "background"; - } else { - os << "normal"; - } - return os; -} } // namespace kernel diff --git a/src/kernel/chain.h b/src/kernel/chain.h index 5afa51f4dda0..4145a1527c0a 100644 --- a/src/kernel/chain.h +++ b/src/kernel/chain.h @@ -7,8 +7,6 @@ #include -#include - class CBlock; class CBlockIndex; class CBlockUndo; @@ -33,10 +31,8 @@ struct BlockInfo { } // namespace interfaces namespace kernel { -struct ChainstateRole; //! Return data from block index. interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* block_index, const CBlock* data = nullptr); -std::ostream& operator<<(std::ostream& os, const ChainstateRole& role); } // namespace kernel #endif // BITCOIN_KERNEL_CHAIN_H diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 806c064ab42c..5d96e5e3bb0d 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -584,17 +584,6 @@ std::unique_ptr CChainParams::TestNet4() return std::make_unique(); } -std::vector CChainParams::GetAvailableSnapshotHeights() const -{ - std::vector heights; - heights.reserve(m_assumeutxo_data.size()); - - for (const auto& data : m_assumeutxo_data) { - heights.emplace_back(data.height); - } - return heights; -} - std::optional GetNetworkForMagic(const MessageStartChars& message) { const auto mainnet_msg = CChainParams::Main()->MessageStart(); diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index f7209bee1b06..7fa5df82812f 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -11,8 +11,6 @@ #include #include #include -#include -#include #include #include @@ -22,32 +20,6 @@ #include #include -struct AssumeutxoHash : public BaseHash { - explicit AssumeutxoHash(const uint256& hash) : BaseHash(hash) {} -}; - -/** - * Holds configuration for use during UTXO snapshot load and validation. The contents - * here are security critical, since they dictate which UTXO snapshots are recognized - * as valid. - */ -struct AssumeutxoData { - int height; - - //! The expected hash of the deserialized UTXO set. - AssumeutxoHash hash_serialized; - - //! Used to populate the m_chain_tx_count value, which is used during BlockManager::LoadBlockIndex(). - //! - //! We need to hardcode the value here because this is computed cumulatively using block data, - //! which we do not necessarily have at the time of snapshot load. - uint64_t m_chain_tx_count; - - //! The hash of the base block for this snapshot. Used to refer to assumeutxo data - //! prior to having a loaded blockindex. - uint256 blockhash; -}; - /** * Holds various statistics on transactions within a chain. Used to estimate * verification progress during chain sync. @@ -89,7 +61,6 @@ class CChainParams const Consensus::Params& GetConsensus() const { return consensus; } const MessageStartChars& MessageStart() const { return pchMessageStart; } uint16_t GetDefaultPort() const { return nDefaultPort; } - std::vector GetAvailableSnapshotHeights() const; const CBlock& GenesisBlock() const { return genesis; } /** Default value for -checkmempool and -checkblockindex argument */ @@ -116,15 +87,6 @@ class CChainParams const std::vector& FixedSeeds() const { return vFixedSeeds; } const HeadersSyncParams& HeadersSync() const { return m_headers_sync_params; } - std::optional AssumeutxoForHeight(int height) const - { - return FindFirst(m_assumeutxo_data, [&](const auto& d) { return d.height == height; }); - } - std::optional AssumeutxoForBlockhash(const uint256& blockhash) const - { - return FindFirst(m_assumeutxo_data, [&](const auto& d) { return d.blockhash == blockhash; }); - } - const ChainTxData& TxData() const { return chainTxData; } /** @@ -177,7 +139,6 @@ class CChainParams std::vector vFixedSeeds; bool fDefaultConsistencyChecks; bool m_is_mockable_chain; - std::vector m_assumeutxo_data; ChainTxData chainTxData; HeadersSyncParams m_headers_sync_params; }; diff --git a/src/kernel/types.h b/src/kernel/types.h deleted file mode 100644 index 164d4cfda81d..000000000000 --- a/src/kernel/types.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -//! @file kernel/types.h is a home for simple enum and struct type definitions -//! that can be used internally by functions in the libbitcoin_kernel library, -//! but also used externally by node, wallet, and GUI code. -//! -//! This file is intended to define only simple types that do not have external -//! dependencies. More complicated types should be defined in dedicated header -//! files. - -#ifndef BITCOIN_KERNEL_TYPES_H -#define BITCOIN_KERNEL_TYPES_H - -namespace kernel { -//! Information about chainstate that notifications are sent from. -struct ChainstateRole { - //! Whether this is a notification from a chainstate that's been fully - //! validated starting from the genesis block. False if it is from an - //! assumeutxo snapshot chainstate that has not been validated yet. - bool validated{true}; - - //! Whether this is a historical chainstate downloading old blocks to - //! validate an assumeutxo snapshot, not syncing to the network tip. - bool historical{false}; -}; -} // namespace kernel - -#endif // BITCOIN_KERNEL_TYPES_H diff --git a/src/net.h b/src/net.h index d0171c5345f1..9bc1f7da35a2 100644 --- a/src/net.h +++ b/src/net.h @@ -1662,9 +1662,7 @@ class CConnman * * This data is replicated in each Peer instance we create. * - * This data is not marked const, but after being set it should not - * change. Unless AssumeUTXO is started, in which case, the peer - * will be limited until the background chain sync finishes. + * This data is not marked const, but after being set it should not change. * * \sa Peer::our_services */ diff --git a/src/net_processing.cpp b/src/net_processing.cpp index a972af790bbf..f9720b02c2af 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -84,7 +83,6 @@ #include #include -using kernel::ChainstateRole; using namespace util::hex_literals; TRACEPOINT_SEMAPHORE(net, inbound_message); @@ -502,7 +500,7 @@ class PeerManagerImpl final : public PeerManager /** Overridden from CValidationInterface. */ void ActiveTipChange(const CBlockIndex& new_tip, bool) override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); - void BlockConnected(const ChainstateRole& role, const std::shared_ptr& pblock, const CBlockIndex* pindexConnected) override + void BlockConnected(const std::shared_ptr& pblock, const CBlockIndex* pindexConnected) override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); void BlockDisconnected(const std::shared_ptr &block, const CBlockIndex* pindex) override EXCLUSIVE_LOCKS_REQUIRED(!m_tx_download_mutex); @@ -906,7 +904,6 @@ class PeerManagerImpl final : public PeerManager void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Request blocks for the background chainstate, if one is in use. */ - void TryDownloadingHistoricalBlocks(const Peer& peer, unsigned int count, std::vector& vBlocks, const CBlockIndex* from_tip, const CBlockIndex* target_block) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** * \brief Find next blocks to download from a peer after a starting block. @@ -1348,17 +1345,6 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co return; } - // When syncing with AssumeUtxo and the snapshot has not yet been validated, - // abort downloading blocks from peers that don't have the snapshot block in their best chain. - // We can't reorg to this chain due to missing undo data until validation completes, - // so downloading blocks from it would be futile. - const CBlockIndex* snap_base{m_chainman.CurrentChainstate().SnapshotBase()}; - if (snap_base && m_chainman.CurrentChainstate().m_assumeutxo == Assumeutxo::UNVALIDATED && - state->pindexBestKnownBlock->GetAncestor(snap_base->nHeight) != snap_base) { - LogDebug(BCLog::NET, "Not downloading blocks from peer=%d, which doesn't have the snapshot block in its best chain.\n", peer.m_id); - return; - } - // Determine the forking point between the peer's chain and our chain: // pindexLastCommonBlock is required to be an ancestor of pindexBestKnownBlock, and will be used as a starting point. // It is being set to the fork point between the peer's best known block and the current tip, unless it is already set to @@ -1381,35 +1367,6 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co FindNextBlocks(vBlocks, peer, state, pindexWalk, count, nWindowEnd, &m_chainman.ActiveChain(), &nodeStaller); } -void PeerManagerImpl::TryDownloadingHistoricalBlocks(const Peer& peer, unsigned int count, std::vector& vBlocks, const CBlockIndex *from_tip, const CBlockIndex* target_block) -{ - Assert(from_tip); - Assert(target_block); - - if (vBlocks.size() >= count) { - return; - } - - vBlocks.reserve(count); - CNodeState *state = Assert(State(peer.m_id)); - - if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->GetAncestor(target_block->nHeight) != target_block) { - // This peer can't provide us the complete series of blocks leading up to the - // assumeutxo snapshot base. - // - // Presumably this peer's chain has less work than our ActiveChain()'s tip, or else we - // will eventually crash when we try to reorg to it. Let other logic - // deal with whether we disconnect this peer. - // - // TODO at some point in the future, we might choose to request what blocks - // this peer does have from the historical chain, despite it not having a - // complete history beneath the snapshot base. - return; - } - - FindNextBlocks(vBlocks, peer, state, from_tip, count, std::min(from_tip->nHeight + BLOCK_DOWNLOAD_WINDOW, target_block->nHeight)); -} - void PeerManagerImpl::FindNextBlocks(std::vector& vBlocks, const Peer& peer, CNodeState *state, const CBlockIndex *pindexWalk, unsigned int count, int nWindowEnd, const CChain* activeChain, NodeId* nodeStaller) { std::vector vToFetch; @@ -2004,12 +1961,9 @@ void PeerManagerImpl::ActiveTipChange(const CBlockIndex& new_tip, bool is_ibd) * possibly reduce dynamic block stalling timeout. */ void PeerManagerImpl::BlockConnected( - const ChainstateRole& role, const std::shared_ptr& pblock, const CBlockIndex* pindex) { - // Update this for all chainstate roles so that we don't mistakenly see peers - // helping us do background IBD as having a stale tip. m_last_tip_update = GetTime(); // In case the dynamic timeout was doubled once or more, reduce it slowly back to its default value @@ -2022,10 +1976,8 @@ void PeerManagerImpl::BlockConnected( } } - // The following task can be skipped since we don't maintain a mempool for - // the historical chainstate, or during ibd since we don't receive incoming - // transactions from peers into the mempool. - if (!role.historical && !m_chainman.IsInitialBlockDownload()) { + // Skip during ibd since we don't receive incoming transactions from peers into the mempool. + if (!m_chainman.IsInitialBlockDownload()) { LOCK(m_tx_download_mutex); m_txdownloadman.BlockConnected(pblock); } @@ -5939,20 +5891,7 @@ bool PeerManagerImpl::SendMessages(CNode& node) return std::max(0, MAX_BLOCKS_IN_TRANSIT_PER_PEER - static_cast(state.vBlocksInFlight.size())); }; - // If there are multiple chainstates, download blocks for the - // current chainstate first, to prioritize getting to network tip - // before downloading historical blocks. FindNextBlocksToDownload(peer, get_inflight_budget(), vToDownload, staller); - auto historical_blocks{m_chainman.GetHistoricalBlockRange()}; - if (historical_blocks && !IsLimitedPeer(peer)) { - // If the first needed historical block is not an ancestor of the last, - // we need to start requesting blocks from their last common ancestor. - const CBlockIndex* from_tip = LastCommonAncestor(historical_blocks->first, historical_blocks->second); - TryDownloadingHistoricalBlocks( - peer, - get_inflight_budget(), - vToDownload, from_tip, historical_blocks->second); - } for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(peer); vGetData.emplace_back(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash()); diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index b0842a005059..ff498d39ca5c 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -60,10 +59,6 @@ static constexpr uint8_t DB_BLOCK_INDEX{'b'}; static constexpr uint8_t DB_FLAG{'F'}; static constexpr uint8_t DB_REINDEX_FLAG{'R'}; static constexpr uint8_t DB_LAST_BLOCK{'l'}; -// Keys used in previous version that might still be found in the DB: -// BlockTreeDB::DB_TXINDEX_BLOCK{'T'}; -// BlockTreeDB::DB_TXINDEX{'t'} -// BlockTreeDB::ReadFlag("txindex") bool BlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo& info) { @@ -314,8 +309,8 @@ void BlockManager::FindFilesToPruneManual( setFilesToPrune.insert(fileNumber); count++; } - LogInfo("[%s] Prune (Manual): prune_height=%d removed %d blk/rev pairs", - chain.GetRole(), last_block_can_prune, count); + LogInfo("Prune (Manual): prune_height=%d removed %d blk/rev pairs", + last_block_can_prune, count); } void BlockManager::FindFilesToPrune( @@ -326,15 +321,9 @@ void BlockManager::FindFilesToPrune( { LOCK2(cs_main, cs_LastBlockFile); // Compute `target` value with maximum size (in bytes) of blocks below the - // `last_prune` height which should be preserved and not pruned. The - // `target` value will be derived from the -prune preference provided by the - // user. If there is a historical chainstate being used to populate indexes - // and validate the snapshot, the target is divided by two so half of the - // block storage will be reserved for the historical chainstate, and the - // other half will be reserved for the most-work chainstate. - const int num_chainstates{chainman.HistoricalChainstate() ? 2 : 1}; + // `last_prune` height which should be preserved and not pruned. const auto target = std::max( - MIN_DISK_SPACE_FOR_BLOCK_FILES, GetPruneTarget() / num_chainstates); + MIN_DISK_SPACE_FOR_BLOCK_FILES, GetPruneTarget()); const uint64_t target_sync_height = chainman.m_best_header->nHeight; if (chain.m_chain.Height() < 0 || target == 0) { @@ -393,8 +382,8 @@ void BlockManager::FindFilesToPrune( } } - LogDebug(BCLog::PRUNE, "[%s] target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n", - chain.GetRole(), target / 1_MiB, nCurrentUsage / 1_MiB, + LogDebug(BCLog::PRUNE, "target=%dMiB actual=%dMiB diff=%dMiB min_height=%d max_prune_height=%d removed %d blk/rev pairs\n", + target / 1_MiB, nCurrentUsage / 1_MiB, (int64_t(target) - int64_t(nCurrentUsage)) / int64_t(1_MiB), min_block_to_prune, last_block_can_prune, count); } @@ -426,37 +415,13 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return pindex; } -bool BlockManager::LoadBlockIndex(const std::optional& snapshot_blockhash) +bool BlockManager::LoadBlockIndex() { if (!m_block_tree_db->LoadBlockIndexGuts( GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); }, m_interrupt)) { return false; } - if (snapshot_blockhash) { - const std::optional maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash); - if (!maybe_au_data) { - m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString())); - return false; - } - const AssumeutxoData& au_data = *Assert(maybe_au_data); - m_snapshot_height = au_data.height; - CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)}; - - // Since m_chain_tx_count (responsible for estimated progress) isn't persisted - // to disk, we must bootstrap the value for assumedvalid chainstates - // from the hardcoded assumeutxo chainparams. - base->m_chain_tx_count = au_data.m_chain_tx_count; - LogInfo("[snapshot] set m_chain_tx_count=%d for %s", au_data.m_chain_tx_count, snapshot_blockhash->ToString()); - } else { - // If this isn't called with a snapshot blockhash, make sure the cached snapshot height - // is null. This is relevant during snapshot completion, when the blockman may be loaded - // with a height that then needs to be cleared after the snapshot is fully validated. - m_snapshot_height.reset(); - } - - Assert(m_snapshot_height.has_value() == snapshot_blockhash.has_value()); - // Calculate nChainWork std::vector vSortedByHeight{GetAllBlockIndices()}; std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), @@ -473,17 +438,11 @@ bool BlockManager::LoadBlockIndex(const std::optional& snapshot_blockha pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); pindex->nTimeMax = (pindex->pprev ? std::max(pindex->pprev->nTimeMax, pindex->nTime) : pindex->nTime); - // We can link the chain of blocks for which we've received transactions at some point, or - // blocks that are assumed-valid on the basis of snapshot load (see - // PopulateAndValidateSnapshot()). - // Pruned nodes may have deleted the block. + // We can link the chain of blocks for which we've received transactions + // at some point. Pruned nodes may have deleted the block. if (pindex->nTx > 0) { if (pindex->pprev) { - if (m_snapshot_height && pindex->nHeight == *m_snapshot_height && - pindex->GetBlockHash() == *snapshot_blockhash) { - // Should have been set above; don't disturb it with code below. - Assert(pindex->m_chain_tx_count > 0); - } else if (pindex->pprev->m_chain_tx_count > 0) { + if (pindex->pprev->m_chain_tx_count > 0) { pindex->m_chain_tx_count = pindex->pprev->m_chain_tx_count + pindex->nTx; } else { pindex->m_chain_tx_count = 0; @@ -532,9 +491,9 @@ void BlockManager::WriteBlockIndexDB() m_block_tree_db->WriteBatchSync(vFiles, max_blockfile, vBlocks); } -bool BlockManager::LoadBlockIndexDB(const std::optional& snapshot_blockhash) +bool BlockManager::LoadBlockIndexDB() { - if (!LoadBlockIndex(snapshot_blockhash)) { + if (!LoadBlockIndex()) { return false; } int max_blockfile_num{0}; @@ -572,11 +531,10 @@ bool BlockManager::LoadBlockIndexDB(const std::optional& snapshot_block } { - // Initialize the blockfile cursors. + // Initialize the blockfile cursor. LOCK(cs_LastBlockFile); - for (size_t i = 0; i < m_blockfile_info.size(); ++i) { - const auto last_height_in_file = m_blockfile_info[i].nHeightLast; - m_blockfile_cursors[BlockfileTypeForHeight(last_height_in_file)] = {static_cast(i), 0}; + if (!m_blockfile_info.empty()) { + m_blockfile_cursor = {static_cast(m_blockfile_info.size() - 1), 0}; } } @@ -774,26 +732,10 @@ bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finali return success; } -BlockfileType BlockManager::BlockfileTypeForHeight(int height) -{ - if (!m_snapshot_height) { - return BlockfileType::NORMAL; - } - return (height >= *m_snapshot_height) ? BlockfileType::ASSUMED : BlockfileType::NORMAL; -} - bool BlockManager::FlushChainstateBlockFile(int tip_height) { LOCK(cs_LastBlockFile); - auto& cursor = m_blockfile_cursors[BlockfileTypeForHeight(tip_height)]; - // If the cursor does not exist, it means an assumeutxo snapshot is loaded, - // but no blocks past the snapshot height have been written yet, so there - // is no data associated with the chainstate, and it is safe not to flush. - if (cursor) { - return FlushBlockFile(cursor->file_num, /*fFinalize=*/false, /*finalize_undo=*/false); - } - // No need to log warnings in this case. - return true; + return FlushBlockFile(m_blockfile_cursor.file_num, /*fFinalize=*/false, /*finalize_undo=*/false); } uint64_t BlockManager::CalculateCurrentUsage() @@ -840,16 +782,7 @@ FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int n { LOCK(cs_LastBlockFile); - const BlockfileType chain_type = BlockfileTypeForHeight(nHeight); - - if (!m_blockfile_cursors[chain_type]) { - // If a snapshot is loaded during runtime, we may not have initialized this cursor yet. - assert(chain_type == BlockfileType::ASSUMED); - const auto new_cursor = BlockfileCursor{this->MaxBlockfileNum() + 1}; - m_blockfile_cursors[chain_type] = new_cursor; - LogDebug(BCLog::BLOCKSTORAGE, "[%s] initializing blockfile cursor to %s\n", chain_type, new_cursor); - } - const int last_blockfile = m_blockfile_cursors[chain_type]->file_num; + const int last_blockfile = m_blockfile_cursor.file_num; int nFile = last_blockfile; if (static_cast(m_blockfile_info.size()) <= nFile) { @@ -874,12 +807,12 @@ FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int n // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it finalize_undo = (static_cast(m_blockfile_info[nFile].nHeightLast) == - Assert(m_blockfile_cursors[chain_type])->undo_height); + m_blockfile_cursor.undo_height); // Try the next unclaimed blockfile number nFile = this->MaxBlockfileNum() + 1; // Set to increment MaxBlockfileNum() for next iteration - m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; + m_blockfile_cursor = BlockfileCursor{nFile}; if (static_cast(m_blockfile_info.size()) <= nFile) { m_blockfile_info.resize(nFile + 1); @@ -906,7 +839,7 @@ FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int n last_blockfile, finalize_undo, nFile); } // No undo data yet in the new file, so reset our undo-height tracking. - m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; + m_blockfile_cursor = BlockfileCursor{nFile}; } m_blockfile_info[nFile].AddBlock(nHeight, nTime); @@ -931,10 +864,8 @@ void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, co LOCK(cs_LastBlockFile); // Update the cursor so it points to the last file. - const BlockfileType chain_type{BlockfileTypeForHeight(nHeight)}; - auto& cursor{m_blockfile_cursors[chain_type]}; - if (!cursor || cursor->file_num < pos.nFile) { - m_blockfile_cursors[chain_type] = BlockfileCursor{pos.nFile}; + if (m_blockfile_cursor.file_num < pos.nFile) { + m_blockfile_cursor = BlockfileCursor{pos.nFile}; } // Update the file information with the current block. @@ -973,8 +904,8 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP bool BlockManager::WriteBlockUndo(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) { AssertLockHeld(::cs_main); - const BlockfileType type = BlockfileTypeForHeight(block.nHeight); - auto& cursor = *Assert(WITH_LOCK(cs_LastBlockFile, return m_blockfile_cursors[type])); + LOCK(cs_LastBlockFile); + auto& cursor{m_blockfile_cursor}; // Write undo information to disk if (block.GetUndoPos().IsNull()) { @@ -1321,15 +1252,6 @@ void ImportBlocks(ChainstateManager& chainman, std::span import_ // End scope of ImportingNow } -std::ostream& operator<<(std::ostream& os, const BlockfileType& type) { - switch(type) { - case BlockfileType::NORMAL: os << "normal"; break; - case BlockfileType::ASSUMED: os << "assumed"; break; - default: os.setstate(std::ios_base::failbit); - } - return os; -} - std::ostream& operator<<(std::ostream& os, const BlockfileCursor& cursor) { os << strprintf("BlockfileCursor(file_num=%d, undo_height=%d)", cursor.file_num, cursor.undo_height); return os; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 0857ba4c8a8d..59977a452591 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -152,15 +152,6 @@ struct PruneLockInfo { int height_first{std::numeric_limits::max()}; }; -enum BlockfileType { - // Values used as array indexes - do not change carelessly. - NORMAL = 0, - ASSUMED = 1, - NUM_TYPES = 2, -}; - -std::ostream& operator<<(std::ostream& os, const BlockfileType& type); - struct BlockfileCursor { // The latest blockfile number. int file_num{0}; @@ -205,7 +196,7 @@ class BlockManager * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral * collections like m_dirty_blockindex. */ - bool LoadBlockIndex(const std::optional& snapshot_blockhash) + bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** Return false if block file or undo file flushing fails. */ @@ -259,27 +250,12 @@ class BlockManager RecursiveMutex cs_LastBlockFile; - //! Since assumedvalid chainstates may be syncing a range of the chain that is very - //! far away from the normal/background validation process, we should segment blockfiles - //! for assumed chainstates. Otherwise, we might have wildly different height ranges - //! mixed into the same block files, which would impair our ability to prune - //! effectively. - //! - //! This data structure maintains separate blockfile number cursors for each - //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindNextBlockPos(). - //! - //! The first element is the NORMAL cursor, second is ASSUMED. - std::array, BlockfileType::NUM_TYPES> - m_blockfile_cursors GUARDED_BY(cs_LastBlockFile) = { - BlockfileCursor{}, - std::nullopt, - }; + //! The current blockfile cursor used for appending new blocks. + BlockfileCursor m_blockfile_cursor GUARDED_BY(cs_LastBlockFile){}; + int MaxBlockfileNum() const EXCLUSIVE_LOCKS_REQUIRED(cs_LastBlockFile) { - static const BlockfileCursor empty_cursor; - const auto& normal = m_blockfile_cursors[BlockfileType::NORMAL].value_or(empty_cursor); - const auto& assumed = m_blockfile_cursors[BlockfileType::ASSUMED].value_or(empty_cursor); - return std::max(normal.file_num, assumed.file_num); + return m_blockfile_cursor.file_num; } /** Global flag to indicate we should check to see if there are @@ -300,8 +276,6 @@ class BlockManager */ std::unordered_map m_prune_locks GUARDED_BY(::cs_main); - BlockfileType BlockfileTypeForHeight(int height); - const kernel::BlockManagerOpts m_opts; const FlatFileSeq m_block_file_seq; @@ -335,20 +309,6 @@ class BlockManager BlockMap m_block_index GUARDED_BY(cs_main); - /** - * The height of the base block of an assumeutxo snapshot, if one is in use. - * - * This controls how blockfiles are segmented by chainstate type to avoid - * comingling different height regions of the chain when an assumedvalid chainstate - * is in use. If heights are drastically different in the same blockfile, pruning - * suffers. - * - * This is set during ActivateSnapshot() or upon LoadBlockIndex() if a snapshot - * had been previously loaded. After the snapshot is validated, this is unset to - * restore normal LoadBlockIndex behavior. - */ - std::optional m_snapshot_height; - std::vector GetAllBlockIndices() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** @@ -360,7 +320,7 @@ class BlockManager std::unique_ptr m_block_tree_db GUARDED_BY(::cs_main); void WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - bool LoadBlockIndexDB(const std::optional& snapshot_blockhash) + bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 1725fe70bd93..e576aaa9085a 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -73,13 +73,6 @@ static ChainstateLoadResult CompleteChainstateInitialization( assert(chainman.m_total_coinstip_cache > 0); assert(chainman.m_total_coinsdb_cache > 0); - // If running with multiple chainstates, limit the cache sizes with a - // discount factor. If discounted the actual cache size will be - // recalculated by `chainman.MaybeRebalanceCaches()`. The discount factor - // is conservatively chosen such that the sum of the caches does not exceed - // the allowable amount during this temporary initialization state. - double init_cache_fraction = chainman.HistoricalChainstate() ? 0.2 : 1.0; - // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! @@ -88,7 +81,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( try { chainstate->InitCoinsDB( - /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, + /*cache_size_bytes=*/chainman.m_total_coinsdb_cache, /*in_memory=*/options.coins_db_in_memory, /*should_wipe=*/options.wipe_chainstate_db); } catch (dbwrapper_error& err) { @@ -114,7 +107,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction); + chainstate->InitCoinsCache(chainman.m_total_coinstip_cache); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(*chainstate)) { @@ -140,11 +133,6 @@ static ChainstateLoadResult CompleteChainstateInitialization( chainman.GetConsensus().SegwitHeight)}; }; - // Now that chainstates are loaded and we're able to flush to - // disk, rebalance the coins caches to desired levels based - // on the condition of each chainstate. - chainman.MaybeRebalanceCaches(); - return {ChainstateLoadStatus::SUCCESS, {}}; } @@ -172,68 +160,14 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize chainman.m_total_coinstip_cache = cache_sizes.coins; chainman.m_total_coinsdb_cache = cache_sizes.coins_db; - // Load the fully validated chainstate. - Chainstate& validated_cs{chainman.InitializeChainstate(options.mempool)}; - - // Load a chain created from a UTXO snapshot, if any exist. - Chainstate* assumeutxo_cs{chainman.LoadAssumeutxoChainstate()}; - - if (assumeutxo_cs && options.wipe_chainstate_db) { - // Reset chainstate target to network tip instead of snapshot block. - validated_cs.SetTargetBlock(nullptr); - LogInfo("[snapshot] deleting snapshot chainstate due to reindexing"); - if (!chainman.DeleteChainstate(*assumeutxo_cs)) { - return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Couldn't remove snapshot chainstate.")}; - } - assumeutxo_cs = nullptr; - } + // Load the chainstate. + chainman.InitializeChainstate(options.mempool); auto [init_status, init_error] = CompleteChainstateInitialization(chainman, options); if (init_status != ChainstateLoadStatus::SUCCESS) { return {init_status, init_error}; } - // If a snapshot chainstate was fully validated by a background chainstate during - // the last run, detect it here and clean up the now-unneeded background - // chainstate. - // - // Why is this cleanup done here (on subsequent restart) and not just when the - // snapshot is actually validated? Because this entails unusual - // filesystem operations to move leveldb data directories around, and that seems - // too risky to do in the middle of normal runtime. - auto snapshot_completion{assumeutxo_cs - ? chainman.MaybeValidateSnapshot(validated_cs, *assumeutxo_cs) - : SnapshotCompletionResult::SKIPPED}; - - if (snapshot_completion == SnapshotCompletionResult::SKIPPED) { - // do nothing; expected case - } else if (snapshot_completion == SnapshotCompletionResult::SUCCESS) { - LogInfo("[snapshot] cleaning up unneeded background chainstate, then reinitializing"); - if (!chainman.ValidatedSnapshotCleanup(validated_cs, *assumeutxo_cs)) { - return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Background chainstate cleanup failed unexpectedly.")}; - } - - // Because ValidatedSnapshotCleanup() has torn down chainstates with - // ChainstateManager::ResetChainstates(), reinitialize them here without - // duplicating the blockindex work above. - assert(chainman.m_chainstates.empty()); - - chainman.InitializeChainstate(options.mempool); - - // A reload of the block index is required to recompute setBlockIndexCandidates - // for the fully validated chainstate. - chainman.ActiveChainstate().ClearBlockIndexCandidates(); - - auto [init_status, init_error] = CompleteChainstateInitialization(chainman, options); - if (init_status != ChainstateLoadStatus::SUCCESS) { - return {init_status, init_error}; - } - } else { - return {ChainstateLoadStatus::FAILURE_FATAL, _( - "UTXO snapshot failed to validate. " - "Restart to resume normal initial block download, or try loading a different snapshot.")}; - } - return {ChainstateLoadStatus::SUCCESS, {}}; } diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 347147f37a68..3fa950c68f72 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -79,7 +79,6 @@ using interfaces::MakeSignalHandler; using interfaces::Mining; using interfaces::Node; using interfaces::Rpc; -using kernel::ChainstateRole; using node::BlockAssembler; using node::BlockWaitOptions; using node::CoinbaseTx; @@ -420,9 +419,9 @@ class NotificationsProxy : public CValidationInterface { m_notifications->transactionRemovedFromMempool(tx, reason); } - void BlockConnected(const ChainstateRole& role, const std::shared_ptr& block, const CBlockIndex* index) override + void BlockConnected(const std::shared_ptr& block, const CBlockIndex* index) override { - m_notifications->blockConnected(role, kernel::MakeBlockInfo(index, block.get())); + m_notifications->blockConnected(kernel::MakeBlockInfo(index, block.get())); } void BlockDisconnected(const std::shared_ptr& block, const CBlockIndex* index) override { @@ -432,9 +431,9 @@ class NotificationsProxy : public CValidationInterface { m_notifications->updatedBlockTip(); } - void ChainStateFlushed(const ChainstateRole& role, const CBlockLocator& locator) override + void ChainStateFlushed(const CBlockLocator& locator) override { - m_notifications->chainStateFlushed(role, locator); + m_notifications->chainStateFlushed(locator); } std::shared_ptr m_notifications; }; @@ -778,12 +777,6 @@ class ChainImpl : public Chain notifications.transactionAddedToMempool(entry.GetSharedTx()); } } - bool hasAssumedValidChain() override - { - LOCK(::cs_main); - return bool{chainman().CurrentChainstate().m_from_snapshot_blockhash}; - } - NodeContext* context() override { return &m_node; } ArgsManager& args() { return *Assert(m_node.args); } ChainstateManager& chainman() { return *Assert(m_node.chainman); } diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp deleted file mode 100644 index 4b61a76c6eb6..000000000000 --- a/src/node/utxo_snapshot.cpp +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright (c) 2022-present The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -namespace node { - -bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) -{ - AssertLockHeld(::cs_main); - assert(snapshot_chainstate.m_from_snapshot_blockhash); - - const std::optional chaindir = snapshot_chainstate.StoragePath(); - assert(chaindir); // Sanity check that chainstate isn't in-memory. - const fs::path write_to = *chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; - - FILE* file{fsbridge::fopen(write_to, "wb")}; - AutoFile afile{file}; - if (afile.IsNull()) { - LogError("[snapshot] failed to open base blockhash file for writing: %s", - fs::PathToString(write_to)); - return false; - } - afile << *snapshot_chainstate.m_from_snapshot_blockhash; - - if (afile.fclose() != 0) { - LogError("[snapshot] failed to close base blockhash file %s after writing", - fs::PathToString(write_to)); - return false; - } - return true; -} - -std::optional ReadSnapshotBaseBlockhash(fs::path chaindir) -{ - if (!fs::exists(chaindir)) { - LogWarning("[snapshot] cannot read base blockhash: no chainstate dir " - "exists at path %s", fs::PathToString(chaindir)); - return std::nullopt; - } - const fs::path read_from = chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; - const std::string read_from_str = fs::PathToString(read_from); - - if (!fs::exists(read_from)) { - LogWarning("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " - "exists at path %s. Try deleting %s and calling loadtxoutset again?", - fs::PathToString(chaindir), read_from_str); - return std::nullopt; - } - - uint256 base_blockhash; - FILE* file{fsbridge::fopen(read_from, "rb")}; - AutoFile afile{file}; - if (afile.IsNull()) { - LogWarning("[snapshot] failed to open base blockhash file for reading: %s", - read_from_str); - return std::nullopt; - } - afile >> base_blockhash; - - int64_t position = afile.tell(); - afile.seek(0, SEEK_END); - if (position != afile.tell()) { - LogWarning("[snapshot] unexpected trailing data in %s", read_from_str); - } - return base_blockhash; -} - -std::optional FindAssumeutxoChainstateDir(const fs::path& data_dir) -{ - fs::path possible_dir = - data_dir / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); - - if (fs::exists(possible_dir)) { - return possible_dir; - } - return std::nullopt; -} - -} // namespace node diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h deleted file mode 100644 index d8b3ca616c84..000000000000 --- a/src/node/utxo_snapshot.h +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-present The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H -#define BITCOIN_NODE_UTXO_SNAPSHOT_H - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -// UTXO set snapshot magic bytes -static constexpr std::array SNAPSHOT_MAGIC_BYTES = {'u', 't', 'x', 'o', 0xff}; - -class Chainstate; - -namespace node { -//! Metadata describing a serialized version of a UTXO set from which an -//! assumeutxo Chainstate can be constructed. -//! All metadata fields come from an untrusted file, so must be validated -//! before being used. Thus, new fields should be added only if needed. -class SnapshotMetadata -{ - inline static const uint16_t VERSION{2}; - const std::set m_supported_versions{VERSION}; - const MessageStartChars m_network_magic; -public: - //! The hash of the block that reflects the tip of the chain for the - //! UTXO set contained in this snapshot. - uint256 m_base_blockhash; - - - //! The number of coins in the UTXO set contained in this snapshot. Used - //! during snapshot load to estimate progress of UTXO set reconstruction. - uint64_t m_coins_count = 0; - - SnapshotMetadata( - const MessageStartChars network_magic) : - m_network_magic(network_magic) { } - SnapshotMetadata( - const MessageStartChars network_magic, - const uint256& base_blockhash, - uint64_t coins_count) : - m_network_magic(network_magic), - m_base_blockhash(base_blockhash), - m_coins_count(coins_count) { } - - template - inline void Serialize(Stream& s) const { - s << SNAPSHOT_MAGIC_BYTES; - s << VERSION; - s << m_network_magic; - s << m_base_blockhash; - s << m_coins_count; - } - - template - inline void Unserialize(Stream& s) { - // Read the snapshot magic bytes - std::array snapshot_magic; - s >> snapshot_magic; - if (snapshot_magic != SNAPSHOT_MAGIC_BYTES) { - throw std::ios_base::failure("Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format."); - } - - // Read the version - uint16_t version; - s >> version; - if (!m_supported_versions.contains(version)) { - throw std::ios_base::failure(strprintf("Version of snapshot %s does not match any of the supported versions.", version)); - } - - // Read the network magic (pchMessageStart) - MessageStartChars message; - s >> message; - if (!std::equal(message.begin(), message.end(), m_network_magic.data())) { - auto metadata_network{GetNetworkForMagic(message)}; - if (metadata_network) { - std::string network_string{ChainTypeToString(metadata_network.value())}; - auto node_network{GetNetworkForMagic(m_network_magic)}; - std::string node_network_string{ChainTypeToString(node_network.value())}; - throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, node_network_string)); - } else { - throw std::ios_base::failure("This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption."); - } - } - - s >> m_base_blockhash; - s >> m_coins_count; - } -}; - -//! The file in the snapshot chainstate dir which stores the base blockhash. This is -//! needed to reconstruct snapshot chainstates on init. -//! -//! Because we only allow loading a single snapshot at a time, there will only be one -//! chainstate directory with this filename present within it. -const fs::path SNAPSHOT_BLOCKHASH_FILENAME{"base_blockhash"}; - -//! Write out the blockhash of the snapshot base block that was used to construct -//! this chainstate. This value is read in during subsequent initializations and -//! used to reconstruct snapshot-based chainstates. -bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - -//! Read the blockhash of the snapshot base block that was used to construct the -//! chainstate. -std::optional ReadSnapshotBaseBlockhash(fs::path chaindir) - EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - -//! Suffix appended to the chainstate (leveldb) dir when created based upon -//! a snapshot. -constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; - - -//! Return a path to the snapshot-based chainstate dir, if one exists. -std::optional FindAssumeutxoChainstateDir(const fs::path& data_dir); - -} // namespace node - -#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 5c053300e70d..51f9960305e2 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include @@ -69,9 +68,14 @@ using interfaces::BlockRef; using interfaces::Mining; using node::BlockManager; using node::NodeContext; -using node::SnapshotMetadata; using util::MakeUnorderedList; +namespace { +// UTXO set file format header written by the dumptxoutset RPC. +constexpr std::array UTXO_DUMP_MAGIC_BYTES = {'u', 't', 'x', 'o', 0xff}; +constexpr uint16_t UTXO_DUMP_VERSION{2}; +} // namespace + std::tuple, CCoinsStats, const CBlockIndex*> PrepareUTXOSnapshot( Chainstate& chainstate, @@ -1274,15 +1278,6 @@ RPCMethod getblockchaininfo() {RPCResult::Type::NUM_TIME, "mediantime", "the median block time expressed in " + UNIX_EPOCH_TIME}, {RPCResult::Type::NUM, "verificationprogress", "estimate of verification progress [0..1]"}, {RPCResult::Type::BOOL, "initialblockdownload", "(debug information) estimate of whether this node is in Initial Block Download mode"}, - {RPCResult::Type::OBJ, "backgroundvalidation", /*optional=*/true, "state info regarding background validation process", - { - {RPCResult::Type::NUM, "snapshotheight", "the height of the snapshot block. Background validation verifies the chain from genesis up to this height"}, - {RPCResult::Type::NUM, "blocks", "the height of the most-work background fully-validated chain. The genesis block has height 0"}, - {RPCResult::Type::STR, "bestblockhash", "the hash of the currently best block validated in the background"}, - {RPCResult::Type::NUM_TIME, "mediantime", "the median block time expressed in " + UNIX_EPOCH_TIME}, - {RPCResult::Type::NUM, "verificationprogress", "estimate of background verification progress [0..1]"}, - {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in background validated chain, in hexadecimal"}, - }}, {RPCResult::Type::STR_HEX, "chainwork", "total amount of work in active chain, in hexadecimal"}, {RPCResult::Type::NUM, "size_on_disk", "the estimated size of the block and undo files on disk"}, {RPCResult::Type::BOOL, "pruned", "if the blocks are subject to pruning"}, @@ -1323,19 +1318,6 @@ RPCMethod getblockchaininfo() obj.pushKV("mediantime", tip.GetMedianTimePast()); obj.pushKV("verificationprogress", chainman.GuessVerificationProgress(&tip)); obj.pushKV("initialblockdownload", chainman.IsInitialBlockDownload()); - auto historical_blocks{chainman.GetHistoricalBlockRange()}; - if (historical_blocks) { - UniValue background_validation(UniValue::VOBJ); - const CBlockIndex& btip{*CHECK_NONFATAL(historical_blocks->first)}; - const CBlockIndex& btarget{*CHECK_NONFATAL(historical_blocks->second)}; - background_validation.pushKV("snapshotheight", btarget.nHeight); - background_validation.pushKV("blocks", btip.nHeight); - background_validation.pushKV("bestblockhash", btip.GetBlockHash().GetHex()); - background_validation.pushKV("mediantime", btip.GetMedianTimePast()); - background_validation.pushKV("chainwork", btip.nChainWork.GetHex()); - background_validation.pushKV("verificationprogress", chainman.GetBackgroundVerificationProgress(btip)); - obj.pushKV("backgroundvalidation", std::move(background_validation)); - } obj.pushKV("chainwork", tip.nChainWork.GetHex()); obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); obj.pushKV("pruned", chainman.m_blockman.IsPruneMode()); @@ -2628,9 +2610,7 @@ class TemporaryPruneLock }; /** - * Serialize the UTXO set to a file for loading elsewhere. - * - * @see SnapshotMetadata + * Serialize the UTXO set to a file. */ static RPCMethod dumptxoutset() { @@ -2959,9 +2939,12 @@ UniValue WriteUTXOSnapshot( tip->nHeight, tip->GetBlockHash().ToString(), fs::PathToString(path), fs::PathToString(temppath))); - SnapshotMetadata metadata{chainstate.m_chainman.GetParams().MessageStart(), tip->GetBlockHash(), maybe_stats->coins_count}; - - afile << metadata; + // Write file header. + afile << UTXO_DUMP_MAGIC_BYTES; + afile << UTXO_DUMP_VERSION; + afile << chainstate.m_chainman.GetParams().MessageStart(); + afile << tip->GetBlockHash(); + afile << maybe_stats->coins_count; COutPoint key; Txid last_hash; diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index a9449c9966be..2897ec3f64aa 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -160,7 +160,6 @@ static const CRPCConvertParam vRPCConvertParams[] = { "verifymessage", 1, "signature", ParamFormat::STRING }, { "verifymessage", 2, "message", ParamFormat::STRING }, { "echoipc", 0, "arg", ParamFormat::STRING }, - { "loadtxoutset", 0, "path", ParamFormat::STRING }, { "signmessagewithprivkey", 1, "message", ParamFormat::STRING }, }; // clang-format on diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index f8b6e20875c2..b147b85d36d5 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -21,6 +21,7 @@ #include #include #include +#include void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, std::optional rbf) { diff --git a/src/test/chainstate_write_tests.cpp b/src/test/chainstate_write_tests.cpp index d0925b6192f6..026d82cefc6d 100644 --- a/src/test/chainstate_write_tests.cpp +++ b/src/test/chainstate_write_tests.cpp @@ -9,8 +9,6 @@ #include -using kernel::ChainstateRole; - // Taken from validation.cpp static constexpr auto DATABASE_WRITE_INTERVAL_MIN{50min}; static constexpr auto DATABASE_WRITE_INTERVAL_MAX{70min}; @@ -21,7 +19,7 @@ BOOST_FIXTURE_TEST_CASE(chainstate_write_interval, TestingSetup) { struct TestSubscriber final : CValidationInterface { bool m_did_flush{false}; - void ChainStateFlushed(const ChainstateRole&, const CBlockLocator&) override + void ChainStateFlushed(const CBlockLocator&) override { m_did_flush = true; } @@ -59,7 +57,7 @@ BOOST_FIXTURE_TEST_CASE(write_during_multiblock_activation, TestChain100Setup) { const CBlockIndex* m_tip{nullptr}; const CBlockIndex* m_flushed_at_block{nullptr}; - void ChainStateFlushed(const ChainstateRole&, const CBlockLocator&) override + void ChainStateFlushed(const CBlockLocator&) override { m_flushed_at_block = m_tip; } diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index d85e7d471db4..78ec16c03e89 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #include #include #include @@ -34,7 +33,6 @@ #include using kernel::CBlockFileInfo; -using node::SnapshotMetadata; void initialize_deserialize() { @@ -305,11 +303,6 @@ FUZZ_TARGET_DESERIALIZE(blocktransactionsrequest_deserialize, { BlockTransactionsRequest btr; DeserializeFromFuzzingInput(buffer, btr); }) -FUZZ_TARGET_DESERIALIZE(snapshotmetadata_deserialize, { - auto msg_start = Params().MessageStart(); - SnapshotMetadata snapshot_metadata{msg_start}; - DeserializeFromFuzzingInput(buffer, snapshot_metadata); -}) FUZZ_TARGET_DESERIALIZE(uint160_deserialize, { uint160 u160; DeserializeFromFuzzingInput(buffer, u160); diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index fe05b9361b5a..772067086382 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -116,7 +116,6 @@ const std::vector RPC_COMMANDS_SAFE_FOR_FUZZING{ "getblockstats", "getblocktemplate", "getchaintips", - "getchainstates", "getchaintxstats", "getconnectioncount", "getdeploymentinfo", diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h deleted file mode 100644 index db11d5657d64..000000000000 --- a/src/test/util/chainstate.h +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2021-present The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. -// -#ifndef BITCOIN_TEST_UTIL_CHAINSTATE_H -#define BITCOIN_TEST_UTIL_CHAINSTATE_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; - -/** - * Create and activate a UTXO snapshot, optionally providing a function to - * malleate the snapshot. - * - * If `reset_chainstate` is true, reset the original chainstate back to the genesis - * block. This allows us to simulate more realistic conditions in which a snapshot is - * loaded into an otherwise mostly-uninitialized datadir. It also allows us to test - * conditions that would otherwise cause shutdowns based on the IBD chainstate going - * past the snapshot it generated. - */ -template -static bool -CreateAndActivateUTXOSnapshot( - TestingSetup* fixture, - F malleation = NoMalleation, - bool reset_chainstate = false, - bool in_memory_chainstate = false) -{ - node::NodeContext& node = fixture->m_node; - fs::path root = fixture->m_path_root; - - // Write out a snapshot to the test's tempdir. - // - int height; - WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); - fs::path snapshot_path = root / fs::u8path(tfm::format("test_snapshot.%d.dat", height)); - FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; - AutoFile auto_outfile{outfile}; - - UniValue result = CreateUTXOSnapshot(node, - node.chainman->ActiveChainstate(), - std::move(auto_outfile), // Will close auto_outfile. - snapshot_path, - snapshot_path); - LogInfo("Wrote UTXO snapshot to %s: %s", - fs::PathToString(snapshot_path.make_preferred()), result.write()); - - // Read the written snapshot in and then activate it. - // - FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; - AutoFile auto_infile{infile}; - node::SnapshotMetadata metadata{node.chainman->GetParams().MessageStart()}; - auto_infile >> metadata; - - malleation(auto_infile, metadata); - - if (reset_chainstate) { - { - // What follows is code to selectively reset chainstate data without - // disturbing the existing BlockManager instance, which is needed to - // recognize the headers chain previously generated by the chainstate we're - // removing. Without those headers, we can't activate the snapshot below. - // - // This is a stripped-down version of node::LoadChainstate which - // preserves the block index. - LOCK(::cs_main); - CBlockIndex *orig_tip = node.chainman->ActiveChainstate().m_chain.Tip(); - uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash(); - node.chainman->ResetChainstates(); - node.chainman->InitializeChainstate(node.mempool.get()); - Chainstate& chain = node.chainman->ActiveChainstate(); - Assert(chain.LoadGenesisBlock()); - // These cache values will be corrected shortly in `MaybeRebalanceCaches`. - chain.InitCoinsDB(1_MiB, /*in_memory=*/true, /*should_wipe=*/false); - chain.InitCoinsCache(1_MiB); - chain.CoinsTip().SetBestBlock(gen_hash); - chain.LoadChainTip(); - node.chainman->MaybeRebalanceCaches(); - - // Reset the HAVE_DATA flags below the snapshot height, simulating - // never-having-downloaded them in the first place. - // TODO: perhaps we could improve this by using pruning to delete - // these blocks instead - CBlockIndex *pindex = orig_tip; - while (pindex && pindex != chain.m_chain.Tip()) { - // Remove all data and validity flags by just setting - // BLOCK_VALID_TREE. Also reset transaction counts and sequence - // ids that are set when blocks are received, to make test setup - // more realistic and satisfy consistency checks in - // CheckBlockIndex(). - assert(pindex->IsValid(BlockStatus::BLOCK_VALID_TREE)); - pindex->nStatus = BlockStatus::BLOCK_VALID_TREE; - pindex->nTx = 0; - pindex->m_chain_tx_count = 0; - pindex->nSequenceId = 0; - pindex = pindex->pprev; - } - chain.PopulateBlockIndexCandidates(); - } - BlockValidationState state; - if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) { - throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); - } - Assert( - 0 == WITH_LOCK(node.chainman->GetMutex(), return node.chainman->ActiveHeight())); - } - - auto& new_active = node.chainman->ActiveChainstate(); - auto* tip = new_active.m_chain.Tip(); - - // Disconnect a block so that the snapshot chainstate will be ahead, otherwise - // it will refuse to activate. - // - // TODO this is a unittest-specific hack, and we should probably rethink how to - // better generate/activate snapshots in unittests. - if (tip->pprev) { - new_active.m_chain.SetTip(*(tip->pprev)); - } - - auto res = node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate); - - // Restore the old tip. - new_active.m_chain.SetTip(*tip); - return !!res; -} - - -#endif // BITCOIN_TEST_UTIL_CHAINSTATE_H diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp index 7e09597ea34e..138fe144f906 100644 --- a/src/test/util/validation.cpp +++ b/src/test/util/validation.cpp @@ -10,8 +10,6 @@ #include #include -using kernel::ChainstateRole; - void TestBlockManager::CleanupForFuzzing() { m_dirty_blockindex.clear(); @@ -44,12 +42,11 @@ void TestChainstateManager::JumpOutOfIbd() } void ValidationInterfaceTest::BlockConnected( - const ChainstateRole& role, CValidationInterface& obj, const std::shared_ptr& block, const CBlockIndex* pindex) { - obj.BlockConnected(role, block, pindex); + obj.BlockConnected(block, pindex); } void TestChainstateManager::InvalidBlockFound(CBlockIndex* pindex, const BlockValidationState& state) { diff --git a/src/test/util/validation.h b/src/test/util/validation.h index 1be3f6a64db6..b261950acd50 100644 --- a/src/test/util/validation.h +++ b/src/test/util/validation.h @@ -35,7 +35,6 @@ class ValidationInterfaceTest { public: static void BlockConnected( - const kernel::ChainstateRole& role, CValidationInterface& obj, const std::shared_ptr& block, const CBlockIndex* pindex); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 083a32da9633..09a6f85e0d2f 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -20,7 +20,6 @@ #include -using kernel::ChainstateRole; using node::BlockAssembler; namespace validation_block_tests { @@ -45,7 +44,7 @@ struct TestSubscriber final : public CValidationInterface { BOOST_CHECK_EQUAL(m_expected_tip, pindexNew->GetBlockHash()); } - void BlockConnected(const ChainstateRole& role, const std::shared_ptr& block, const CBlockIndex* pindex) override + void BlockConnected(const std::shared_ptr& block, const CBlockIndex* pindex) override { BOOST_CHECK_EQUAL(m_expected_tip, block->hashPrevBlock); BOOST_CHECK_EQUAL(m_expected_tip, pindex->pprev->GetBlockHash()); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 004a71325dc8..e0c84d52efbd 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -10,7 +10,6 @@ #include #include