index: Coinstats index can be activated with command line flag

Summary:
This is a backport of [[bitcoin/bitcoin#19521 | core#19521]] [5/17]
bitcoin/bitcoin@3c914d5
partial  bitcoin/bitcoin@6a4c0c0

The functional test only checks that "-coinstatsindex" argument does not break anything. The rest of the functional test from commit 6a4c0c09ab is not yet applicable and will be added in the next commit.

This includes also minor documentation fixups from [[ bitcoin/bitcoin#21818 | core#21818]]

Depends on D11598 and D11595

Test Plan: `ninja all check-all`

Reviewers: #bitcoin_abc, Fabien

Reviewed By: #bitcoin_abc, Fabien

Subscribers: Fabien

Differential Revision: https://reviews.bitcoinabc.org/D11599

contrib/debian/examples/bitcoin.conf

@@ -127,8 +127,11 @@
 # be validated sooner.
 #paytxfee=0.00
 
+# Maintain coinstats index used by the gettxoutsetinfo RPC (default: 0).
+#coinstatsindex=1
+
 # Enable pruning to reduce storage requirements by deleting old blocks.
-# This mode is incompatible with -txindex and -rescan.
+# This mode is incompatible with -txindex, -coinstatsindex and -rescan.
 # 0 = default (no pruning).
 # 1 = allows manual pruning via RPC.
 # >=550 = target to stay under in MiB.

doc/release-notes.md

@@ -5,3 +5,5 @@ Bitcoin ABC version 0.25.8 is now available from:
   <https://download.bitcoinabc.org/0.25.8/>
 
 This release includes the following features and fixes:
+- Users can start their node with the option `-coinstatsindex` which syncs an
+  index of coin statistics in the background.

src/init.cpp

@@ -29,6 +29,7 @@
 #include <httprpc.h>
 #include <httpserver.h>
 #include <index/blockfilterindex.h>
+#include <index/coinstatsindex.h>
 #include <index/txindex.h>
 #include <interfaces/chain.h>
 #include <interfaces/node.h>
@@ -175,6 +176,9 @@ void Interrupt(NodeContext &node) {
         g_txindex->Interrupt();
     }
     ForEachBlockFilterIndex([](BlockFilterIndex &index) { index.Interrupt(); });
+    if (g_coin_stats_index) {
+        g_coin_stats_index->Interrupt();
+    }
 }
 
 void Shutdown(NodeContext &node) {
@@ -269,6 +273,10 @@ void Shutdown(NodeContext &node) {
         g_txindex->Stop();
         g_txindex.reset();
     }
+    if (g_coin_stats_index) {
+        g_coin_stats_index->Stop();
+        g_coin_stats_index.reset();
+    }
     ForEachBlockFilterIndex([](BlockFilterIndex &index) { index.Stop(); });
     DestroyAllBlockFilterIndexes();
 
@@ -467,6 +475,11 @@ void SetupServerArgs(NodeContext &node) {
                   " not affected. (default: %u)",
                   DEFAULT_BLOCKSONLY),
         ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-coinstatsindex",
+                   strprintf("Maintain coinstats index used by the "
+                             "gettxoutsetinfo RPC (default: %u)",
+                             DEFAULT_COINSTATSINDEX),
+                   ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg(
         "-conf=<file>",
         strprintf("Specify path to read-only configuration file. Relative "
@@ -567,11 +580,12 @@ void SetupServerArgs(NodeContext &node) {
                   "of old blocks. This allows the pruneblockchain RPC to be "
                   "called to delete specific blocks, and enables automatic "
                   "pruning of old blocks if a target size in MiB is provided. "
-                  "This mode is incompatible with -txindex and -rescan. "
-                  "Warning: Reverting this setting requires re-downloading the "
-                  "entire blockchain. (default: 0 = disable pruning blocks, 1 "
-                  "= allow manual pruning via RPC, >=%u = automatically prune "
-                  "block files to stay under the specified target size in MiB)",
+                  "This mode is incompatible with -txindex, -coinstatsindex "
+                  "and -rescan. Warning: Reverting this setting requires "
+                  "re-downloading the entire blockchain. (default: 0 = disable "
+                  "pruning blocks, 1 = allow manual pruning via RPC, >=%u = "
+                  "automatically prune block files to stay under the specified "
+                  "target size in MiB)",
                   MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024),
         ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg(
@@ -1846,11 +1860,15 @@ bool AppInitParameterInteraction(Config &config, const ArgsManager &args) {
         nLocalServices = ServiceFlags(nLocalServices | NODE_COMPACT_FILTERS);
     }
 
-    // if using block pruning, then disallow txindex
+    // if using block pruning, then disallow txindex and coinstatsindex
     if (args.GetArg("-prune", 0)) {
         if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
             return InitError(_("Prune mode is incompatible with -txindex."));
         }
+        if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) {
+            return InitError(
+                _("Prune mode is incompatible with -coinstatsindex."));
+        }
     }
 
     // -bind and -whitebind can't be set when not listening
@@ -2855,6 +2873,11 @@ bool AppInitMain(Config &config, RPCServer &rpcServer,
         GetBlockFilterIndex(filter_type)->Start();
     }
 
+    if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) {
+        g_coin_stats_index = std::make_unique<CoinStatsIndex>(
+            /* cache size */ 0, false, fReindex);
+        g_coin_stats_index->Start();
+    }
     // Step 9: load wallet
     for (const auto &client : node.chain_clients) {
         if (!client->load()) {

src/node/coinstats.cpp

@@ -8,6 +8,7 @@
 #include <coins.h>
 #include <crypto/muhash.h>
 #include <hash.h>
+#include <index/coinstatsindex.h>
 #include <primitives/txid.h>
 #include <serialize.h>
 #include <util/check.h>
@@ -91,15 +92,24 @@ static bool GetUTXOStats(CCoinsView *view, BlockManager &blockman,
                          const std::function<void()> &interruption_point) {
     std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
     assert(pcursor);
-
     stats.hashBlock = pcursor->GetBestBlock();
+
+    const CBlockIndex *pindex;
     {
         LOCK(cs_main);
         assert(std::addressof(g_chainman.m_blockman) ==
                std::addressof(blockman));
 
-        const CBlockIndex *block = blockman.LookupBlockIndex(stats.hashBlock);
-        stats.nHeight = Assert(block)->nHeight;
+        pindex = blockman.LookupBlockIndex(stats.hashBlock);
+        stats.nHeight = Assert(pindex)->nHeight;
+    }
+
+    // Use CoinStatsIndex if it is available and a hash_type of Muhash or None
+    // was requested
+    if ((stats.m_hash_type == CoinStatsHashType::MUHASH ||
+         stats.m_hash_type == CoinStatsHashType::NONE) &&
+        g_coin_stats_index) {
+        return g_coin_stats_index->LookUpStats(pindex, stats);
     }
 
     PrepareHash(hash_obj, stats);

src/rpc/blockchain.cpp

@@ -16,6 +16,7 @@
 #include <core_io.h>
 #include <hash.h>
 #include <index/blockfilterindex.h>
+#include <index/coinstatsindex.h>
 #include <node/blockstorage.h>
 #include <node/coinstats.h>
 #include <node/context.h>
@@ -1321,6 +1322,19 @@ static RPCHelpMan gettxoutsetinfo() {
                 ret.pushKV("disk_size", stats.nDiskSize);
                 ret.pushKV("total_amount", stats.nTotalAmount);
             } else {
+                if (g_coin_stats_index) {
+                    const IndexSummary summary{
+                        g_coin_stats_index->GetSummary()};
+
+                    if (!summary.synced) {
+                        throw JSONRPCError(
+                            RPC_INTERNAL_ERROR,
+                            strprintf("Unable to read UTXO set because "
+                                      "coinstatsindex is still syncing. "
+                                      "Current height: %d",
+                                      summary.best_block_height));
+                    }
+                }
                 throw JSONRPCError(RPC_INTERNAL_ERROR,
                                    "Unable to read UTXO set");
             }

src/validation.h

@@ -89,6 +89,7 @@ static const int DEFAULT_SCRIPTCHECK_THREADS = 0;
 static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60;
 static const bool DEFAULT_CHECKPOINTS_ENABLED = true;
 static const bool DEFAULT_TXINDEX = false;
+static constexpr bool DEFAULT_COINSTATSINDEX{false};
 static const char *const DEFAULT_BLOCKFILTERINDEX = "0";
 
 /** Default for -persistmempool */

test/functional/feature_coinstatsindex.py

@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+# Copyright (c) 2020 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test coinstatsindex across nodes.
+
+Test that the values returned by gettxoutsetinfo are consistent
+between a node running the coinstatsindex and a node without
+the index.
+"""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, try_rpc
+
+
+class CoinStatsIndexTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.supports_cli = False
+        self.extra_args = [
+            [],
+            ["-coinstatsindex"]
+        ]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        self._test_coin_stats_index()
+
+    def _test_coin_stats_index(self):
+        node = self.nodes[0]
+        index_node = self.nodes[1]
+        # Both none and muhash options allow the usage of the index
+        index_hash_options = ['none', 'muhash']
+
+        # Generate a normal transaction and mine it
+        node.generate(101)
+        address = self.nodes[0].get_deterministic_priv_key().address
+        node.sendtoaddress(
+            address=address,
+            amount=10_000_000,
+            subtractfeefromamount=True)
+        node.generate(1)
+
+        self.sync_blocks(timeout=120)
+
+        self.log.info(
+            "Test that gettxoutsetinfo() output is consistent with or without coinstatsindex option")
+        self.wait_until(lambda: not try_rpc(-32603,
+                        "Unable to read UTXO set", node.gettxoutsetinfo))
+        res0 = node.gettxoutsetinfo('none')
+
+        # The fields 'disk_size' and 'transactions' do not work on the index, so
+        # don't check them.
+        del res0['disk_size'], res0['transactions']
+
+        self.wait_until(lambda: not try_rpc(-32603,
+                                            "Unable to read UTXO set",
+                                            index_node.gettxoutsetinfo,
+                                            'muhash'))
+        for hash_option in index_hash_options:
+            res1 = index_node.gettxoutsetinfo(hash_option)
+            res1.pop('muhash', None)
+
+            # The fields 'disk_size' and 'transactions' do not work on the index
+            # so don't check them (they will be removed from the index in the
+            # next commit).
+            del res1['disk_size'], res1['transactions']
+
+            # Everything left should be the same
+            assert_equal(res1, res0)
+
+
+if __name__ == '__main__':
+    CoinStatsIndexTest().main()

test/functional/test_framework/test_framework.py

@@ -794,8 +794,8 @@ def cache_path(*paths):
             # Remove empty wallets dir
             os.rmdir(cache_path('wallets'))
             for entry in os.listdir(cache_path()):
-                # Only keep chainstate and blocks folder
-                if entry not in ['chainstate', 'blocks']:
+                # Only keep indexes, chainstate and blocks folders
+                if entry not in ['chainstate', 'blocks', 'indexes']:
                     os.remove(cache_path(entry))
 
         for i in range(self.num_nodes):

test/lint/lint-circular-dependencies.sh

@@ -17,6 +17,7 @@ EXPECTED_CIRCULAR_DEPENDENCIES=(
     "node/blockstorage -> validation -> node/blockstorage"
     "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex"
     "index/base -> validation -> index/blockfilterindex -> index/base"
+    "index/coinstatsindex -> node/coinstats -> index/coinstatsindex"
     "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel"
     "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui"
     "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel"