validation: assumeutxo params for testnet and signet

contrib/devtools/test_utxo_snapshots.sh

@@ -0,0 +1,200 @@
+#!/usr/bin/env bash
+# Demonstrate the creation and usage of UTXO snapshots.
+#
+# A server node starts up, IBDs up to a certain height, then generates a UTXO
+# snapshot at that point.
+#
+# The server then downloads more blocks (to create a diff from the snapshot).
+#
+# We bring a client up, load the UTXO snapshot, and we show the client sync to
+# the "network tip" and then start a background validation of the snapshot it
+# loaded. We see the background validation chainstate removed after validation
+# completes.
+#
+
+export LC_ALL=C
+set -e
+
+BASE_HEIGHT=${1:-30000}
+INCREMENTAL_HEIGHT=20000
+FINAL_HEIGHT=$(($BASE_HEIGHT + $INCREMENTAL_HEIGHT))
+
+SERVER_DATADIR="$(pwd)/utxodemo-data-server-$BASE_HEIGHT"
+CLIENT_DATADIR="$(pwd)/utxodemo-data-client-$BASE_HEIGHT"
+UTXO_DAT_FILE="$(pwd)/utxo.$BASE_HEIGHT.dat"
+
+# Chosen to try to not interfere with any running bitcoind processes.
+SERVER_PORT=8633
+SERVER_RPC_PORT=8632
+
+CLIENT_PORT=8733
+CLIENT_RPC_PORT=8732
+
+SERVER_PORTS="-port=${SERVER_PORT} -rpcport=${SERVER_RPC_PORT}"
+CLIENT_PORTS="-port=${CLIENT_PORT} -rpcport=${CLIENT_RPC_PORT}"
+
+# Ensure the client exercises all indexes to test that snapshot use works
+# properly with indexes.
+ALL_INDEXES="-txindex -coinstatsindex -blockfilterindex=1"
+
+if ! command -v jq >/dev/null ; then
+  echo "This script requires jq to parse JSON RPC output. Please install it."
+  echo "(e.g. sudo apt install jq)"
+  exit 1
+fi
+
+DUMP_OUTPUT="dumptxoutset-output-$BASE_HEIGHT.json"
+
+finish() {
+  echo
+  echo "Killing server and client PIDs ($SERVER_PID, $CLIENT_PID) and cleaning up datadirs"
+  echo
+  rm -f "$UTXO_DAT_FILE" "$DUMP_OUTPUT"
+  rm -rf "$SERVER_DATADIR" "$CLIENT_DATADIR"
+  kill -9 "$SERVER_PID" "$CLIENT_PID"
+}
+
+trap finish EXIT
+
+# Need to specify these to trick client into accepting server as a peer
+# it can IBD from, otherwise the default values prevent IBD from the server node.
+EARLY_IBD_FLAGS="-maxtipage=9223372036854775207 -minimumchainwork=0x00"
+
+server_rpc() {
+  ./src/bitcoin-cli -rpcport=$SERVER_RPC_PORT -datadir="$SERVER_DATADIR" "$@"
+}
+client_rpc() {
+  ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir="$CLIENT_DATADIR" "$@"
+}
+server_sleep_til_boot() {
+  while ! server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+}
+client_sleep_til_boot() {
+  while ! client_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+}
+
+mkdir -p "$SERVER_DATADIR" "$CLIENT_DATADIR"
+
+echo "Hi, welcome to the assumeutxo demo/test"
+echo
+echo "We're going to"
+echo
+echo "  - start up a 'server' node, sync it via mainnet IBD to height ${BASE_HEIGHT}"
+echo "  - create a UTXO snapshot at that height"
+echo "  - IBD ${INCREMENTAL_HEIGHT} more blocks on top of that"
+echo
+echo "then we'll demonstrate assumeutxo by "
+echo
+echo "  - starting another node (the 'client') and loading the snapshot in"
+echo "    * first you'll have to modify the code slightly (chainparams) and recompile"
+echo "    * don't worry, we'll make it easy"
+echo "  - observing the client sync ${INCREMENTAL_HEIGHT} blocks on top of the snapshot from the server"
+echo "  - observing the client validate the snapshot chain via background IBD"
+echo
+read -p "Press [enter] to continue" _
+
+echo
+echo "-- Starting the demo. You might want to run the two following commands in"
+echo "   separate terminal windows:"
+echo
+echo "   watch -n0.1 tail -n 30 $SERVER_DATADIR/debug.log"
+echo "   watch -n0.1 tail -n 30 $CLIENT_DATADIR/debug.log"
+echo
+read -p "Press [enter] to continue" _
+
+echo
+echo "-- IBDing the blocks (height=$BASE_HEIGHT) required to the server node..."
+./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
+    -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -stopatheight="$BASE_HEIGHT" >/dev/null
+
+echo
+echo "-- Creating snapshot at ~ height $BASE_HEIGHT ($UTXO_DAT_FILE)..."
+sleep 2
+./src/bitcoind -logthreadnames=1 $SERVER_PORTS \
+    -datadir="$SERVER_DATADIR" $EARLY_IBD_FLAGS -connect=0 -listen=0 >/dev/null &
+SERVER_PID="$!"
+
+server_sleep_til_boot
+server_rpc dumptxoutset "$UTXO_DAT_FILE" > "$DUMP_OUTPUT"
+cat "$DUMP_OUTPUT"
+kill -9 "$SERVER_PID"
+
+RPC_BASE_HEIGHT=$(jq -r .base_height < "$DUMP_OUTPUT")
+RPC_AU=$(jq -r .txoutset_hash < "$DUMP_OUTPUT")
+RPC_NCHAINTX=$(jq -r .nchaintx < "$DUMP_OUTPUT")
+RPC_BLOCKHASH=$(jq -r .base_hash < "$DUMP_OUTPUT")
+
+# Wait for server to shutdown...
+while server_rpc ping >/dev/null 2>&1; do sleep 0.1; done
+
+echo
+echo "-- Now: add the following to CMainParams::m_assumeutxo_data"
+echo "   in src/kernel/chainparams.cpp, and recompile:"
+echo
+echo "   {${RPC_BASE_HEIGHT}, AssumeutxoHash{uint256S(\"0x${RPC_AU}\")}, ${RPC_NCHAINTX}, uint256S(\"0x${RPC_BLOCKHASH}\")},"
+echo
+echo
+echo "-- IBDing more blocks to the server node (height=$FINAL_HEIGHT) so there is a diff between snapshot and tip..."
+./src/bitcoind $SERVER_PORTS -logthreadnames=1 -datadir="$SERVER_DATADIR" \
+    $EARLY_IBD_FLAGS -stopatheight="$FINAL_HEIGHT" >/dev/null
+
+echo
+echo "-- Starting the server node to provide blocks to the client node..."
+./src/bitcoind $SERVER_PORTS -logthreadnames=1 -debug=net -datadir="$SERVER_DATADIR" \
+    $EARLY_IBD_FLAGS -connect=0 -listen=1 >/dev/null &
+SERVER_PID="$!"
+server_sleep_til_boot
+
+echo
+echo "-- Okay, what you're about to see is the client starting up and activating the snapshot."
+echo "   I'm going to display the top 14 log lines from the client on top of an RPC called"
+echo "   getchainstates, which is like getblockchaininfo but for both the snapshot and "
+echo "   background validation chainstates."
+echo
+echo "   You're going to first see the snapshot chainstate sync to the server's tip, then"
+echo "   the background IBD chain kicks in to validate up to the base of the snapshot."
+echo
+echo "   Once validation of the snapshot is done, you should see log lines indicating"
+echo "   that we've deleted the background validation chainstate."
+echo
+echo "   Once everything completes, exit the watch command with CTRL+C."
+echo
+read -p "When you're ready for all this, hit [enter]" _
+
+echo
+echo "-- Starting the client node to get headers from the server, then load the snapshot..."
+./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" \
+    -connect=0 -addnode=127.0.0.1:$SERVER_PORT -debug=net $EARLY_IBD_FLAGS >/dev/null &
+CLIENT_PID="$!"
+client_sleep_til_boot
+
+echo
+echo "-- Initial state of the client:"
+client_rpc getchainstates
+
+echo
+echo "-- Loading UTXO snapshot into client..."
+client_rpc loadtxoutset "$UTXO_DAT_FILE"
+
+watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
+
+echo
+echo "-- Okay, now I'm going to restart the client to make sure that the snapshot chain reloads "
+echo "   as the main chain properly..."
+echo
+echo "   Press CTRL+C after you're satisfied to exit the demo"
+echo
+read -p "Press [enter] to continue"
+
+while kill -0 "$CLIENT_PID"; do
+    sleep 1
+done
+./src/bitcoind $CLIENT_PORTS $ALL_INDEXES -logthreadnames=1 -datadir="$CLIENT_DATADIR" -connect=0 \
+    -addnode=127.0.0.1:$SERVER_PORT "$EARLY_IBD_FLAGS" >/dev/null &
+CLIENT_PID="$!"
+client_sleep_til_boot
+
+watch -n 0.3 "( tail -n 14 $CLIENT_DATADIR/debug.log ; echo ; ./src/bitcoin-cli -rpcport=$CLIENT_RPC_PORT -datadir=$CLIENT_DATADIR getchainstates) | cat"
+
+echo
+echo "-- Done!"

doc/design/assumeutxo.md

@@ -3,7 +3,7 @@
 Assumeutxo is a feature that allows fast bootstrapping of a validating bitcoind
 instance with a very similar security model to assumevalid.
 
-The RPC commands `dumptxoutset` and `loadtxoutset` (yet to be merged) are used to
+The RPC commands `dumptxoutset` and `loadtxoutset` are used to
 respectively generate and load UTXO snapshots. The utility script
 `./contrib/devtools/utxo_snapshot.sh` may be of use.
 

doc/release-notes-27596.md

@@ -0,0 +1,28 @@
+Pruning
+-------
+
+When using assumeutxo with `-prune`, the prune budget may be exceeded if it is set
+lower than 1100MB (i.e. `MIN_DISK_SPACE_FOR_BLOCK_FILES * 2`). Prune budget is normally
+split evenly across each chainstate, unless the resulting prune budget per chainstate
+is beneath `MIN_DISK_SPACE_FOR_BLOCK_FILES` in which case that value will be used.
+
+RPC
+---
+
+`loadtxoutset` has been added, which allows loading a UTXO snapshot of the format
+generated by `dumptxoutset`. Once this snapshot is loaded, its contents will be
+deserialized into a second chainstate data structure, which is then used to sync to
+the network's tip under a security model very much like `assumevalid`.
+
+Meanwhile, the original chainstate will complete the initial block download process in
+the background, eventually validating up to the block that the snapshot is based upon.
+
+The result is a usable bitcoind instance that is current with the network tip in a
+matter of minutes rather than hours. UTXO snapshot are typically obtained via
+third-party sources (HTTP, torrent, etc.) which is reasonable since their contents
+are always checked by hash.
+
+You can find more information on this process in the `assumeutxo` design
+document (<https://github.com/bitcoin/bitcoin/blob/master/doc/design/assumeutxo.md>).
+
+`getchainstates` has been added to aid in monitoring the assumeutxo sync process.

doc/zmq.md

@@ -113,11 +113,11 @@ Where the 8-byte uints correspond to the mempool sequence number.
     | hashtx | <32-byte transaction hash in Little Endian> | <uint32 sequence number in Little Endian>
 
 
-`rawblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages).
+`rawblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`rawblock`), the second part is the serialized block, and the last part is a sequence number (representing the message count to detect lost messages).
 
     | rawblock | <serialized block> | <uint32 sequence number in Little Endian>
 
-`hashblock`: Notifies when the chain tip is updated. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages).
+`hashblock`: Notifies when the chain tip is updated. When assumeutxo is in use, this notification will not be issued for historical blocks connected to the background validation chainstate. Messages are ZMQ multipart messages with three parts. The first part is the topic (`hashblock`), the second part is the 32-byte block hash, and the last part is a sequence number (representing the message count to detect lost messages).
 
     | hashblock | <32-byte block hash in Little Endian> | <uint32 sequence number in Little Endian>
 

src/bench/wallet_create_tx.cpp

@@ -70,7 +70,7 @@ void generateFakeBlock(const CChainParams& params,
 
     // notify wallet
     const auto& pindex = WITH_LOCK(::cs_main, return context.chainman->ActiveChain().Tip());
-    wallet.blockConnected(kernel::MakeBlockInfo(pindex, &block));
+    wallet.blockConnected(ChainstateRole::NORMAL, kernel::MakeBlockInfo(pindex, &block));
 }
 
 struct PreSelectInputs {

src/chain.h

@@ -276,6 +276,12 @@ class CBlockIndex
      *
      * Does not imply the transactions are consensus-valid (ConnectTip might fail)
      * Does not imply the transactions are still stored on disk. (IsBlockPruned might return true)
+     *
+     * Note that this will be true for the snapshot base block, if one is loaded (and
+     * all subsequent assumed-valid blocks) since its nChainTx value will have been set
+     * manually based on the related AssumeutxoData entry.
+     *
+     * TODO: potentially change the name of this based on the fact above.
      */
     bool HaveTxsDownloaded() const { return nChainTx != 0; }
 

src/index/base.cpp

@@ -79,9 +79,15 @@ BaseIndex::~BaseIndex()
 
 bool BaseIndex::Init()
 {
+    AssertLockNotHeld(cs_main);
+
+    // May need reset if index is being restarted.
+    m_interrupt.reset();
+
     // m_chainstate member gives indexing code access to node internals. It is
     // removed in followup https://github.com/bitcoin/bitcoin/pull/24230
-    m_chainstate = &m_chain->context()->chainman->ActiveChainstate();
+    m_chainstate = WITH_LOCK(::cs_main,
+        return &m_chain->context()->chainman->GetChainstateForIndexing());
     // Register to validation interface before setting the 'm_synced' flag, so that
     // callbacks are not missed once m_synced is true.
     RegisterValidationInterface(this);
@@ -92,7 +98,8 @@ bool BaseIndex::Init()
     }
 
     LOCK(cs_main);
-    CChain& active_chain = m_chainstate->m_chain;
+    CChain& index_chain = m_chainstate->m_chain;
+
     if (locator.IsNull()) {
         SetBestBlockIndex(nullptr);
     } else {
@@ -114,7 +121,7 @@ bool BaseIndex::Init()
     // Note: this will latch to true immediately if the user starts up with an empty
     // datadir and an index enabled. If this is the case, indexation will happen solely
     // via `BlockConnected` signals until, possibly, the next restart.
-    m_synced = start_block == active_chain.Tip();
+    m_synced = start_block == index_chain.Tip();
     m_init = true;
     return true;
 }
@@ -143,6 +150,8 @@ void BaseIndex::ThreadSync()
         std::chrono::steady_clock::time_point last_locator_write_time{0s};
         while (true) {
             if (m_interrupt) {
+                LogPrintf("%s: m_interrupt set; exiting ThreadSync\n", GetName());
+
                 SetBestBlockIndex(pindex);
                 // No need to handle errors in Commit. If it fails, the error will be already be
                 // logged. The best way to recover is to continue, as index cannot be corrupted by
@@ -250,8 +259,19 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti
     return true;
 }
 
-void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
+void BaseIndex::BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex)
 {
+    // Ignore events from the assumed-valid chain; we will process its blocks
+    // (sequentially) after it is fully verified by the background chainstate. This
+    // is to avoid any out-of-order indexing.
+    //
+    // TODO at some point we could parameterize whether a particular index can be
+    // built out of order, but for now just do the conservative simple thing.
+    if (role == ChainstateRole::ASSUMEDVALID) {
+        return;
+    }
+
+    // Ignore BlockConnected signals until we have fully indexed the chain.
     if (!m_synced) {
         return;
     }
@@ -296,8 +316,14 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
     }
 }
 
-void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
+void BaseIndex::ChainStateFlushed(ChainstateRole role, const CBlockLocator& locator)
 {
+    // Ignore events from the assumed-valid chain; we will process its blocks
+    // (sequentially) after it is fully verified by the background chainstate.
+    if (role == ChainstateRole::ASSUMEDVALID) {
+        return;
+    }
+
     if (!m_synced) {
         return;
     }