Add feerate histogram to getmempoolinfo

src/rest.cpp

@@ -31,6 +31,7 @@
 
 #include <any>
 #include <string>
+#include <optional>
 
 #include <univalue.h>
 
@@ -654,7 +655,7 @@ static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::s
         if (param == "contents") {
             str_json = MempoolToJSON(*mempool, true).write() + "\n";
         } else {
-            str_json = MempoolInfoToJSON(*mempool).write() + "\n";
+            str_json = MempoolInfoToJSON(*mempool, std::nullopt).write() + "\n";
         }
 
         req->WriteHeader("Content-Type", "application/json");

src/rpc/client.cpp

@@ -173,6 +173,7 @@ static const CRPCConvertParam vRPCConvertParams[] =
     { "getblockstats", 1, "stats" },
     { "pruneblockchain", 0, "height" },
     { "keypoolrefill", 0, "newsize" },
+    { "getmempoolinfo", 0, "fee_histogram" },
     { "getrawmempool", 0, "verbose" },
     { "getrawmempool", 1, "mempool_sequence" },
     { "estimatesmartfee", 0, "conf_target" },

src/rpc/mempool.cpp

@@ -15,6 +15,7 @@
 #include <policy/rbf.h>
 #include <policy/settings.h>
 #include <primitives/transaction.h>
+#include <rpc/mempool.h>
 #include <rpc/server.h>
 #include <rpc/server_util.h>
 #include <rpc/util.h>
@@ -24,6 +25,7 @@
 #include <util/moneystr.h>
 #include <util/time.h>
 
+#include <optional>
 #include <utility>
 
 using kernel::DumpMempool;
@@ -669,7 +671,7 @@ static RPCHelpMan gettxspendingprevout()
     };
 }
 
-UniValue MempoolInfoToJSON(const CTxMemPool& pool)
+UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional<MempoolHistogramFeeRates>& histogram_floors)
 {
     // Make sure this call is atomic in the pool.
     LOCK(pool.cs);
@@ -685,14 +687,64 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool)
     ret.pushKV("incrementalrelayfee", ValueFromAmount(pool.m_incremental_relay_feerate.GetFeePerK()));
     ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()});
     ret.pushKV("fullrbf", pool.m_full_rbf);
+
+    if (histogram_floors) {
+        const MempoolHistogramFeeRates& floors{histogram_floors.value()};
+
+        std::vector<uint64_t> sizes(floors.size(), 0);
+        std::vector<uint64_t> count(floors.size(), 0);
+        std::vector<CAmount> fees(floors.size(), 0);
+
+        for (const CTxMemPoolEntry& e : pool.mapTx) {
+            const CAmount fee{e.GetFee()};
+            const uint32_t size{uint32_t(e.GetTxSize())};
+            const CAmount fee_rate{CFeeRate{fee, size}.GetFee(1)};
+
+            // Distribute fee rates
+            for (size_t i = floors.size(); i-- > 0;) {
+                if (fee_rate >= floors[i]) {
+                    sizes[i] += size;
+                    ++count[i];
+                    fees[i] += fee;
+                    break;
+                }
+            }
+        }
+
+        // Track total amount of available fees in fee rate groups
+        CAmount total_fees = 0;
+        UniValue groups(UniValue::VOBJ);
+        for (size_t i = 0; i < floors.size(); ++i) {
+            UniValue info_sub(UniValue::VOBJ);
+            info_sub.pushKV("size", sizes.at(i));
+            info_sub.pushKV("count", count.at(i));
+            info_sub.pushKV("fees", fees.at(i));
+            info_sub.pushKV("from", floors.at(i));
+
+            total_fees += fees.at(i);
+            groups.pushKV(ToString(floors.at(i)), info_sub);
+        }
+
+        UniValue info(UniValue::VOBJ);
+        info.pushKV("fee_rate_groups", groups);
+        info.pushKV("total_fees", total_fees);
+        ret.pushKV("fee_histogram", info);
+    }
+
     return ret;
 }
 
 static RPCHelpMan getmempoolinfo()
 {
     return RPCHelpMan{"getmempoolinfo",
-        "Returns details on the active state of the TX memory pool.",
-        {},
+        "Returns details on the active state of the TX memory pool.\n",
+        {
+            {"fee_histogram", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Fee statistics grouped by fee rate ranges",
+                {
+                    {"fee_rate", RPCArg::Type::NUM, RPCArg::Optional::NO, "Fee rate (in " + CURRENCY_ATOM + "/vB) to group the fees by"},
+                },
+            },
+        },
         RPCResult{
             RPCResult::Type::OBJ, "", "",
             {
@@ -707,14 +759,53 @@ static RPCHelpMan getmempoolinfo()
                 {RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or replacement in " + CURRENCY_UNIT + "/kvB"},
                 {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"},
                 {RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"},
+                {RPCResult::Type::OBJ, "fee_histogram", /*optional=*/true, "",
+                    {
+                        {RPCResult::Type::OBJ_DYN, "fee_rate_groups", "",
+                        {
+                            {RPCResult::Type::OBJ, "<fee_rate_group>", "Fee rate group named by its lower bound (in " + CURRENCY_ATOM + "/vB), identical to the \"from\" field below",
+                            {
+                                {RPCResult::Type::NUM, "size", "Cumulative size of all transactions in the fee rate group (in vBytes)"},
+                                {RPCResult::Type::NUM, "count", "Number of transactions in the fee rate group"},
+                                {RPCResult::Type::NUM, "fees", "Cumulative fees of all transactions in the fee rate group (in " + CURRENCY_ATOM + ")"},
+                                {RPCResult::Type::NUM, "from", "Group contains transactions with fee rates equal or greater than this value (in " + CURRENCY_ATOM + "/vB)"},
+                            }}}},
+                        {RPCResult::Type::NUM, "total_fees", "Total available fees in mempool (in " + CURRENCY_ATOM + ")"},
+                    }},
             }},
         RPCExamples{
-            HelpExampleCli("getmempoolinfo", "")
-            + HelpExampleRpc("getmempoolinfo", "")
+            HelpExampleCli("getmempoolinfo", "") +
+            HelpExampleCli("getmempoolinfo", R"("[0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100, 120, 140, 170, 200]")") +
+            HelpExampleRpc("getmempoolinfo", "") +
+            HelpExampleRpc("getmempoolinfo", R"([0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100, 120, 140, 170, 200])")
         },
         [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue
 {
-    return MempoolInfoToJSON(EnsureAnyMemPool(request.context));
+    MempoolHistogramFeeRates histogram_floors;
+    std::optional<MempoolHistogramFeeRates> histogram_floors_opt = std::nullopt;
+
+    if (!request.params[0].isNull()) {
+        const UniValue histogram_floors_univalue = request.params[0].get_array();
+
+        if (histogram_floors_univalue.empty()) {
+            throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid number of parameters");
+        }
+
+        for (size_t i = 0; i < histogram_floors_univalue.size(); ++i) {
+            int64_t value = histogram_floors_univalue[i].getInt<int64_t>();
+
+            if (value < 0) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Non-negative values are expected");
+            } else if (i > 0 && histogram_floors.back() >= value) {
+                throw JSONRPCError(RPC_INVALID_PARAMETER, "Strictly increasing values are expected");
+            }
+
+            histogram_floors.push_back(value);
+        }
+        histogram_floors_opt = std::optional<MempoolHistogramFeeRates>(std::move(histogram_floors));
+    }
+
+    return MempoolInfoToJSON(EnsureAnyMemPool(request.context), histogram_floors_opt);
 },
     };
 }

src/rpc/mempool.h

@@ -5,11 +5,18 @@
 #ifndef BITCOIN_RPC_MEMPOOL_H
 #define BITCOIN_RPC_MEMPOOL_H
 
+#include <consensus/amount.h>
+
+#include <optional>
+#include <vector>
+
 class CTxMemPool;
 class UniValue;
 
+typedef std::vector<CAmount> MempoolHistogramFeeRates;
+
 /** Mempool information to JSON */
-UniValue MempoolInfoToJSON(const CTxMemPool& pool);
+UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional<MempoolHistogramFeeRates>& histogram_floors);
 
 /** Mempool to JSON */
 UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false);

test/functional/mempool_fee_histogram.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+# Copyright (c) 2023 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test mempool fee histogram."""
+
+from decimal import Decimal
+
+from test_framework.blocktools import COINBASE_MATURITY
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal,
+    assert_greater_than,
+    assert_greater_than_or_equal,
+)
+
+class MempoolFeeHistogramTest(BitcoinTestFramework):
+    def add_options(self, parser):
+        self.add_wallet_options(parser)
+
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_wallet()
+
+    def run_test(self):
+        node = self.nodes[0]
+        self.generate(self.nodes[0], COINBASE_MATURITY + 2, sync_fun=self.no_op)
+
+        # We have two UTXOs (utxo_1 and utxo_2) and we create three changeless transactions:
+        # - tx1 (5 sat/vB): spending utxo_1
+        # - tx2 (14 sat/vB): spending output from tx1
+        # - tx3 (6 sat/vB): spending utxo_2 and the output from tx2
+
+        self.log.info("Test getmempoolinfo does not return fee histogram by default")
+        assert ("fee_histogram" not in node.getmempoolinfo())
+
+        self.log.info("Test getmempoolinfo returns empty fee histogram when mempool is empty")
+        info = node.getmempoolinfo([1, 2, 3])
+
+        (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
+        assert_equal(0, non_empty_groups)
+        assert_equal(3, empty_groups)
+        assert_equal(0, total_fees)
+
+        for i in ['1', '2', '3']:
+            assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['size'])
+            assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['count'])
+            assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['fees'])
+            assert_equal(int(i), info['fee_histogram']['fee_rate_groups'][i]['from'])
+
+        self.log.info("Test that we have two spendable UTXOs and lock the second one")
+        utxos = node.listunspent()
+        assert_equal(2, len(utxos))
+        node.lockunspent(False, [{"txid": utxos[1]["txid"], "vout": utxos[1]["vout"]}])
+
+        self.log.info("Send tx1 transaction with 5 sat/vB fee rate")
+        node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("50.0"), fee_rate=5, subtractfeefromamount=True)
+
+        self.log.info("Test fee rate histogram when mempool contains 1 transaction (tx1: 5 sat/vB)")
+        info = node.getmempoolinfo([1, 3, 5, 10])
+        (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
+        assert_equal(1, non_empty_groups)
+        assert_equal(3, empty_groups)
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count'])
+        assert_equal(total_fees, info['fee_histogram']['total_fees'])
+
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['1']['size'])
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['1']['count'])
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['1']['fees'])
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['1']['from'])
+
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['3']['size'])
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['3']['count'])
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['3']['fees'])
+        assert_equal(3, info['fee_histogram']['fee_rate_groups']['3']['from'])
+
+        assert_equal(188, info['fee_histogram']['fee_rate_groups']['5']['size'])
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count'])
+        assert_equal(940, info['fee_histogram']['fee_rate_groups']['5']['fees'])
+        assert_equal(5, info['fee_histogram']['fee_rate_groups']['5']['from'])
+
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['10']['size'])
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['10']['count'])
+        assert_equal(0, info['fee_histogram']['fee_rate_groups']['10']['fees'])
+        assert_equal(10, info['fee_histogram']['fee_rate_groups']['10']['from'])
+
+        self.log.info("Send tx2 transaction with 14 sat/vB fee rate (spends tx1 UTXO)")
+        node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("25.0"), fee_rate=14, subtractfeefromamount=True)
+
+        self.log.info("Test fee rate histogram when mempool contains 2 transactions (tx1: 5 sat/vB, tx2: 14 sat/vB)")
+        info = node.getmempoolinfo([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
+
+        # Verify that tx1 and tx2 are reported in 5 sat/vB and 14 sat/vB in fee rate groups respectively
+        (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
+        assert_equal(2, non_empty_groups)
+        assert_equal(13, empty_groups)
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count'])
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['14']['count'])
+        assert_equal(total_fees, info['fee_histogram']['total_fees'])
+
+        # Unlock the second UTXO which we locked
+        node.lockunspent(True, [{"txid": utxos[1]["txid"], "vout": utxos[1]["vout"]}])
+
+        self.log.info("Send tx3 transaction with 6 sat/vB fee rate (spends all available UTXOs)")
+        node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("99.9"), fee_rate=6, subtractfeefromamount=True)
+
+        self.log.info("Test fee rate histogram when mempool contains 3 transactions (tx1: 5 sat/vB, tx2: 14 sat/vB, tx3: 6 sat/vB)")
+        info = node.getmempoolinfo([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15])
+
+        # Verify that each of 5, 6 and 14 sat/vB fee rate groups contain one transaction
+        (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram'])
+        assert_equal(3, non_empty_groups)
+        assert_equal(12, empty_groups)
+
+        for i in ['1', '2', '3', '4', '7', '8', '9', '10', '11', '12', '13', '15']:
+            assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['size'])
+            assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['count'])
+            assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['fees'])
+            assert_equal(int(i), info['fee_histogram']['fee_rate_groups'][i]['from'])
+
+        assert_equal(188, info['fee_histogram']['fee_rate_groups']['5']['size'])
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count'])
+        assert_equal(940, info['fee_histogram']['fee_rate_groups']['5']['fees'])
+        assert_equal(5, info['fee_histogram']['fee_rate_groups']['5']['from'])
+
+        assert_equal(356, info['fee_histogram']['fee_rate_groups']['6']['size'])
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['6']['count'])
+        assert_equal(2136, info['fee_histogram']['fee_rate_groups']['6']['fees'])
+        assert_equal(6, info['fee_histogram']['fee_rate_groups']['6']['from'])
+
+        assert_equal(141, info['fee_histogram']['fee_rate_groups']['14']['size'])
+        assert_equal(1, info['fee_histogram']['fee_rate_groups']['14']['count'])
+        assert_equal(1974, info['fee_histogram']['fee_rate_groups']['14']['fees'])
+        assert_equal(14, info['fee_histogram']['fee_rate_groups']['14']['from'])
+
+        assert_equal(total_fees, info['fee_histogram']['total_fees'])
+
+    def histogram_stats(self, histogram):
+        total_fees = 0
+        empty_count = 0
+        non_empty_count = 0
+
+        for key, bin in histogram['fee_rate_groups'].items():
+            assert_equal(int(key), bin['from'])
+            if bin['fees'] > 0:
+                assert_greater_than(bin['count'], 0)
+            else:
+                assert_equal(bin['count'], 0)
+            assert_greater_than_or_equal(bin['fees'], 0)
+            assert_greater_than_or_equal(bin['size'], 0)
+            total_fees += bin['fees']
+
+            if bin['count'] == 0:
+                empty_count += 1
+            else:
+                non_empty_count += 1
+
+        return (non_empty_count, empty_count, total_fees)
+
+if __name__ == '__main__':
+    MempoolFeeHistogramTest().main()

test/functional/test_runner.py

@@ -266,6 +266,7 @@
     'p2p_initial_headers_sync.py',
     'feature_nulldummy.py',
     'mempool_accept.py',
+    'mempool_fee_histogram.py',
     'mempool_expiry.py',
     'wallet_import_with_label.py --legacy-wallet',
     'wallet_importdescriptors.py --descriptors',