From d242aa52de7e09ee756e56fe1d6f8941ffd48f06 Mon Sep 17 00:00:00 2001 From: Kiminuo Date: Sat, 6 Mar 2021 20:30:17 +0100 Subject: [PATCH 1/2] Introduce fee histogram in getmempoolinfo RPC command Co-authored-by: Jonas Schnelli Co-authored-by: Jon Atack --- src/rest.cpp | 3 +- src/rpc/client.cpp | 1 + src/rpc/mempool.cpp | 103 +++++++++++++++++++++++++++++++++++++++++--- src/rpc/mempool.h | 9 +++- 4 files changed, 108 insertions(+), 8 deletions(-) diff --git a/src/rest.cpp b/src/rest.cpp index a874f4eb6ddb4..08b83aecc743d 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -31,6 +31,7 @@ #include #include +#include #include @@ -654,7 +655,7 @@ static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::s if (param == "contents") { str_json = MempoolToJSON(*mempool, true).write() + "\n"; } else { - str_json = MempoolInfoToJSON(*mempool).write() + "\n"; + str_json = MempoolInfoToJSON(*mempool, std::nullopt).write() + "\n"; } req->WriteHeader("Content-Type", "application/json"); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 9449b9d197ef7..0ec98a9f4e40a 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -173,6 +173,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getblockstats", 1, "stats" }, { "pruneblockchain", 0, "height" }, { "keypoolrefill", 0, "newsize" }, + { "getmempoolinfo", 0, "fee_histogram" }, { "getrawmempool", 0, "verbose" }, { "getrawmempool", 1, "mempool_sequence" }, { "estimatesmartfee", 0, "conf_target" }, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 3a69e2d8a22cf..520339459e603 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include +#include #include using kernel::DumpMempool; @@ -669,7 +671,7 @@ static RPCHelpMan gettxspendingprevout() }; } -UniValue MempoolInfoToJSON(const CTxMemPool& pool) +UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional& histogram_floors) { // Make sure this call is atomic in the pool. LOCK(pool.cs); @@ -685,14 +687,64 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("incrementalrelayfee", ValueFromAmount(pool.m_incremental_relay_feerate.GetFeePerK())); ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); ret.pushKV("fullrbf", pool.m_full_rbf); + + if (histogram_floors) { + const MempoolHistogramFeeRates& floors{histogram_floors.value()}; + + std::vector sizes(floors.size(), 0); + std::vector count(floors.size(), 0); + std::vector fees(floors.size(), 0); + + for (const CTxMemPoolEntry& e : pool.mapTx) { + const CAmount fee{e.GetFee()}; + const uint32_t size{uint32_t(e.GetTxSize())}; + const CAmount fee_rate{CFeeRate{fee, size}.GetFee(1)}; + + // Distribute fee rates + for (size_t i = floors.size(); i-- > 0;) { + if (fee_rate >= floors[i]) { + sizes[i] += size; + ++count[i]; + fees[i] += fee; + break; + } + } + } + + // Track total amount of available fees in fee rate groups + CAmount total_fees = 0; + UniValue groups(UniValue::VOBJ); + for (size_t i = 0; i < floors.size(); ++i) { + UniValue info_sub(UniValue::VOBJ); + info_sub.pushKV("size", sizes.at(i)); + info_sub.pushKV("count", count.at(i)); + info_sub.pushKV("fees", fees.at(i)); + info_sub.pushKV("from", floors.at(i)); + + total_fees += fees.at(i); + groups.pushKV(ToString(floors.at(i)), info_sub); + } + + UniValue info(UniValue::VOBJ); + info.pushKV("fee_rate_groups", groups); + info.pushKV("total_fees", total_fees); + ret.pushKV("fee_histogram", info); + } + return ret; } static RPCHelpMan getmempoolinfo() { return RPCHelpMan{"getmempoolinfo", - "Returns details on the active state of the TX memory pool.", - {}, + "Returns details on the active state of the TX memory pool.\n", + { + {"fee_histogram", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Fee statistics grouped by fee rate ranges", + { + {"fee_rate", RPCArg::Type::NUM, RPCArg::Optional::NO, "Fee rate (in " + CURRENCY_ATOM + "/vB) to group the fees by"}, + }, + }, + }, RPCResult{ RPCResult::Type::OBJ, "", "", { @@ -707,14 +759,53 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or replacement in " + CURRENCY_UNIT + "/kvB"}, {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"}, {RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"}, + {RPCResult::Type::OBJ, "fee_histogram", /*optional=*/true, "", + { + {RPCResult::Type::OBJ_DYN, "fee_rate_groups", "", + { + {RPCResult::Type::OBJ, "", "Fee rate group named by its lower bound (in " + CURRENCY_ATOM + "/vB), identical to the \"from\" field below", + { + {RPCResult::Type::NUM, "size", "Cumulative size of all transactions in the fee rate group (in vBytes)"}, + {RPCResult::Type::NUM, "count", "Number of transactions in the fee rate group"}, + {RPCResult::Type::NUM, "fees", "Cumulative fees of all transactions in the fee rate group (in " + CURRENCY_ATOM + ")"}, + {RPCResult::Type::NUM, "from", "Group contains transactions with fee rates equal or greater than this value (in " + CURRENCY_ATOM + "/vB)"}, + }}}}, + {RPCResult::Type::NUM, "total_fees", "Total available fees in mempool (in " + CURRENCY_ATOM + ")"}, + }}, }}, RPCExamples{ - HelpExampleCli("getmempoolinfo", "") - + HelpExampleRpc("getmempoolinfo", "") + HelpExampleCli("getmempoolinfo", "") + + HelpExampleCli("getmempoolinfo", R"("[0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100, 120, 140, 170, 200]")") + + HelpExampleRpc("getmempoolinfo", "") + + HelpExampleRpc("getmempoolinfo", R"([0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, 25, 30, 40, 50, 60, 70, 80, 100, 120, 140, 170, 200])") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - return MempoolInfoToJSON(EnsureAnyMemPool(request.context)); + MempoolHistogramFeeRates histogram_floors; + std::optional histogram_floors_opt = std::nullopt; + + if (!request.params[0].isNull()) { + const UniValue histogram_floors_univalue = request.params[0].get_array(); + + if (histogram_floors_univalue.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid number of parameters"); + } + + for (size_t i = 0; i < histogram_floors_univalue.size(); ++i) { + int64_t value = histogram_floors_univalue[i].getInt(); + + if (value < 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Non-negative values are expected"); + } else if (i > 0 && histogram_floors.back() >= value) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Strictly increasing values are expected"); + } + + histogram_floors.push_back(value); + } + histogram_floors_opt = std::optional(std::move(histogram_floors)); + } + + return MempoolInfoToJSON(EnsureAnyMemPool(request.context), histogram_floors_opt); }, }; } diff --git a/src/rpc/mempool.h b/src/rpc/mempool.h index 229d7d52dd0b9..5feb6089afe78 100644 --- a/src/rpc/mempool.h +++ b/src/rpc/mempool.h @@ -5,11 +5,18 @@ #ifndef BITCOIN_RPC_MEMPOOL_H #define BITCOIN_RPC_MEMPOOL_H +#include + +#include +#include + class CTxMemPool; class UniValue; +typedef std::vector MempoolHistogramFeeRates; + /** Mempool information to JSON */ -UniValue MempoolInfoToJSON(const CTxMemPool& pool); +UniValue MempoolInfoToJSON(const CTxMemPool& pool, const std::optional& histogram_floors); /** Mempool to JSON */ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose = false, bool include_mempool_sequence = false); From c5e53d0d21ff166ee9ab21f043d839aa112fcf96 Mon Sep 17 00:00:00 2001 From: Kiminuo Date: Sat, 6 Mar 2021 22:54:05 +0100 Subject: [PATCH 2/2] test: Add mempool fee histogram test coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original commit: https://github.com/bitcoin/bitcoin/commit/0b6ba66238c377116bc6c21e19cffbf1b6dfc788 Co-authored-by: João Barbosa Co-authored-by: Jon Atack --- test/functional/mempool_fee_histogram.py | 164 +++++++++++++++++++++++ test/functional/test_runner.py | 1 + 2 files changed, 165 insertions(+) create mode 100755 test/functional/mempool_fee_histogram.py diff --git a/test/functional/mempool_fee_histogram.py b/test/functional/mempool_fee_histogram.py new file mode 100755 index 0000000000000..cddb37d9ac810 --- /dev/null +++ b/test/functional/mempool_fee_histogram.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# Copyright (c) 2023 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test mempool fee histogram.""" + +from decimal import Decimal + +from test_framework.blocktools import COINBASE_MATURITY +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import ( + assert_equal, + assert_greater_than, + assert_greater_than_or_equal, +) + +class MempoolFeeHistogramTest(BitcoinTestFramework): + def add_options(self, parser): + self.add_wallet_options(parser) + + def set_test_params(self): + self.setup_clean_chain = True + self.num_nodes = 1 + + def skip_test_if_missing_module(self): + self.skip_if_no_wallet() + + def run_test(self): + node = self.nodes[0] + self.generate(self.nodes[0], COINBASE_MATURITY + 2, sync_fun=self.no_op) + + # We have two UTXOs (utxo_1 and utxo_2) and we create three changeless transactions: + # - tx1 (5 sat/vB): spending utxo_1 + # - tx2 (14 sat/vB): spending output from tx1 + # - tx3 (6 sat/vB): spending utxo_2 and the output from tx2 + + self.log.info("Test getmempoolinfo does not return fee histogram by default") + assert ("fee_histogram" not in node.getmempoolinfo()) + + self.log.info("Test getmempoolinfo returns empty fee histogram when mempool is empty") + info = node.getmempoolinfo([1, 2, 3]) + + (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram']) + assert_equal(0, non_empty_groups) + assert_equal(3, empty_groups) + assert_equal(0, total_fees) + + for i in ['1', '2', '3']: + assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['size']) + assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['count']) + assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['fees']) + assert_equal(int(i), info['fee_histogram']['fee_rate_groups'][i]['from']) + + self.log.info("Test that we have two spendable UTXOs and lock the second one") + utxos = node.listunspent() + assert_equal(2, len(utxos)) + node.lockunspent(False, [{"txid": utxos[1]["txid"], "vout": utxos[1]["vout"]}]) + + self.log.info("Send tx1 transaction with 5 sat/vB fee rate") + node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("50.0"), fee_rate=5, subtractfeefromamount=True) + + self.log.info("Test fee rate histogram when mempool contains 1 transaction (tx1: 5 sat/vB)") + info = node.getmempoolinfo([1, 3, 5, 10]) + (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram']) + assert_equal(1, non_empty_groups) + assert_equal(3, empty_groups) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count']) + assert_equal(total_fees, info['fee_histogram']['total_fees']) + + assert_equal(0, info['fee_histogram']['fee_rate_groups']['1']['size']) + assert_equal(0, info['fee_histogram']['fee_rate_groups']['1']['count']) + assert_equal(0, info['fee_histogram']['fee_rate_groups']['1']['fees']) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['1']['from']) + + assert_equal(0, info['fee_histogram']['fee_rate_groups']['3']['size']) + assert_equal(0, info['fee_histogram']['fee_rate_groups']['3']['count']) + assert_equal(0, info['fee_histogram']['fee_rate_groups']['3']['fees']) + assert_equal(3, info['fee_histogram']['fee_rate_groups']['3']['from']) + + assert_equal(188, info['fee_histogram']['fee_rate_groups']['5']['size']) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count']) + assert_equal(940, info['fee_histogram']['fee_rate_groups']['5']['fees']) + assert_equal(5, info['fee_histogram']['fee_rate_groups']['5']['from']) + + assert_equal(0, info['fee_histogram']['fee_rate_groups']['10']['size']) + assert_equal(0, info['fee_histogram']['fee_rate_groups']['10']['count']) + assert_equal(0, info['fee_histogram']['fee_rate_groups']['10']['fees']) + assert_equal(10, info['fee_histogram']['fee_rate_groups']['10']['from']) + + self.log.info("Send tx2 transaction with 14 sat/vB fee rate (spends tx1 UTXO)") + node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("25.0"), fee_rate=14, subtractfeefromamount=True) + + self.log.info("Test fee rate histogram when mempool contains 2 transactions (tx1: 5 sat/vB, tx2: 14 sat/vB)") + info = node.getmempoolinfo([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + + # Verify that tx1 and tx2 are reported in 5 sat/vB and 14 sat/vB in fee rate groups respectively + (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram']) + assert_equal(2, non_empty_groups) + assert_equal(13, empty_groups) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count']) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['14']['count']) + assert_equal(total_fees, info['fee_histogram']['total_fees']) + + # Unlock the second UTXO which we locked + node.lockunspent(True, [{"txid": utxos[1]["txid"], "vout": utxos[1]["vout"]}]) + + self.log.info("Send tx3 transaction with 6 sat/vB fee rate (spends all available UTXOs)") + node.sendtoaddress(address=node.getnewaddress(), amount=Decimal("99.9"), fee_rate=6, subtractfeefromamount=True) + + self.log.info("Test fee rate histogram when mempool contains 3 transactions (tx1: 5 sat/vB, tx2: 14 sat/vB, tx3: 6 sat/vB)") + info = node.getmempoolinfo([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]) + + # Verify that each of 5, 6 and 14 sat/vB fee rate groups contain one transaction + (non_empty_groups, empty_groups, total_fees) = self.histogram_stats(info['fee_histogram']) + assert_equal(3, non_empty_groups) + assert_equal(12, empty_groups) + + for i in ['1', '2', '3', '4', '7', '8', '9', '10', '11', '12', '13', '15']: + assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['size']) + assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['count']) + assert_equal(0, info['fee_histogram']['fee_rate_groups'][i]['fees']) + assert_equal(int(i), info['fee_histogram']['fee_rate_groups'][i]['from']) + + assert_equal(188, info['fee_histogram']['fee_rate_groups']['5']['size']) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['5']['count']) + assert_equal(940, info['fee_histogram']['fee_rate_groups']['5']['fees']) + assert_equal(5, info['fee_histogram']['fee_rate_groups']['5']['from']) + + assert_equal(356, info['fee_histogram']['fee_rate_groups']['6']['size']) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['6']['count']) + assert_equal(2136, info['fee_histogram']['fee_rate_groups']['6']['fees']) + assert_equal(6, info['fee_histogram']['fee_rate_groups']['6']['from']) + + assert_equal(141, info['fee_histogram']['fee_rate_groups']['14']['size']) + assert_equal(1, info['fee_histogram']['fee_rate_groups']['14']['count']) + assert_equal(1974, info['fee_histogram']['fee_rate_groups']['14']['fees']) + assert_equal(14, info['fee_histogram']['fee_rate_groups']['14']['from']) + + assert_equal(total_fees, info['fee_histogram']['total_fees']) + + def histogram_stats(self, histogram): + total_fees = 0 + empty_count = 0 + non_empty_count = 0 + + for key, bin in histogram['fee_rate_groups'].items(): + assert_equal(int(key), bin['from']) + if bin['fees'] > 0: + assert_greater_than(bin['count'], 0) + else: + assert_equal(bin['count'], 0) + assert_greater_than_or_equal(bin['fees'], 0) + assert_greater_than_or_equal(bin['size'], 0) + total_fees += bin['fees'] + + if bin['count'] == 0: + empty_count += 1 + else: + non_empty_count += 1 + + return (non_empty_count, empty_count, total_fees) + +if __name__ == '__main__': + MempoolFeeHistogramTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index f8869ae6bcdcc..f8914149facf2 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -266,6 +266,7 @@ 'p2p_initial_headers_sync.py', 'feature_nulldummy.py', 'mempool_accept.py', + 'mempool_fee_histogram.py', 'mempool_expiry.py', 'wallet_import_with_label.py --legacy-wallet', 'wallet_importdescriptors.py --descriptors',