mining: add getMemoryLoad() and track template non-mempool memory footprint

src/interfaces/mining.h

@@ -92,6 +92,12 @@ class BlockTemplate
     virtual void interruptWait() = 0;
 };
 
+//! Tracks memory usage for template-bound transactions.
+struct MemoryLoad
+{
+    uint64_t usage{0};
+};
+
 //! Interface giving clients (RPC, Stratum v2 Template Provider in the future)
 //! ability to create block templates.
 class Mining
@@ -147,6 +153,12 @@ class Mining
      */
     virtual bool checkBlock(const CBlock& block, const node::BlockCheckOptions& options, std::string& reason, std::string& debug) = 0;
 
+    /**
+     * Returns the current memory load for template transactions outside the
+     * mempool.
+     */
+    virtual MemoryLoad getMemoryLoad() = 0;
+
     //! Get internal node context. Useful for RPC and testing,
     //! but not accessible across processes.
     virtual node::NodeContext* context() { return nullptr; }

src/ipc/capnp/mining.capnp

@@ -19,6 +19,7 @@ interface Mining $Proxy.wrap("interfaces::Mining") {
     waitTipChanged @3 (context :Proxy.Context, currentTip: Data, timeout: Float64) -> (result: Common.BlockRef);
     createNewBlock @4 (options: BlockCreateOptions) -> (result: BlockTemplate);
     checkBlock @5 (block: Data, options: BlockCheckOptions) -> (reason: Text, debug: Text, result: Bool);
+    getMemoryLoad @6 (context :Proxy.Context) -> (result: MemoryLoad);
 }
 
 interface BlockTemplate $Proxy.wrap("interfaces::BlockTemplate") {
@@ -42,6 +43,10 @@ struct BlockCreateOptions $Proxy.wrap("node::BlockCreateOptions") {
     coinbaseOutputMaxAdditionalSigops @2 :UInt64 $Proxy.name("coinbase_output_max_additional_sigops");
 }
 
+struct MemoryLoad $Proxy.wrap("interfaces::MemoryLoad") {
+    usage @0 :UInt64;
+}
+
 struct BlockWaitOptions $Proxy.wrap("node::BlockWaitOptions") {
     timeout @0 : Float64 $Proxy.name("timeout");
     feeThreshold @1 : Int64 $Proxy.name("fee_threshold");

src/node/interfaces.cpp

@@ -24,7 +24,9 @@
 #include <kernel/context.h>
 #include <kernel/mempool_entry.h>
 #include <logging.h>
+#include <logging/timer.h>
 #include <mapport.h>
+#include <memusage.h>
 #include <net.h>
 #include <net_processing.h>
 #include <netaddress.h>
@@ -67,6 +69,7 @@
 #include <any>
 #include <memory>
 #include <optional>
+#include <ranges>
 #include <utility>
 
 #include <boost/signals2/signal.hpp>
@@ -78,6 +81,7 @@ using interfaces::Chain;
 using interfaces::FoundBlock;
 using interfaces::Handler;
 using interfaces::MakeSignalHandler;
+using interfaces::MemoryLoad;
 using interfaces::Mining;
 using interfaces::Node;
 using interfaces::WalletLoader;
@@ -857,16 +861,40 @@ class ChainImpl : public Chain
     NodeContext& m_node;
 };
 
+typedef std::map<CTransactionRef, size_t> TxTemplateMap;
 class BlockTemplateImpl : public BlockTemplate
 {
 public:
-    explicit BlockTemplateImpl(BlockAssembler::Options assemble_options,
-                               std::unique_ptr<CBlockTemplate> block_template,
-                               NodeContext& node) : m_assemble_options(std::move(assemble_options)),
-                                                    m_block_template(std::move(block_template)),
-                                                    m_node(node)
+    explicit BlockTemplateImpl(
+        BlockAssembler::Options assemble_options,
+        std::unique_ptr<CBlockTemplate> block_template,
+        NodeContext& node,
+        std::shared_ptr<TxTemplateMap> tx_template_refs) : m_assemble_options(std::move(assemble_options)),
+                                                          m_block_template(std::move(block_template)),
+                                                          m_node(node),
+                                                          m_tx_template_refs(std::move(tx_template_refs))
+
     {
         assert(m_block_template);
+
+        TxTemplateMap& tx_refs{*Assert(m_tx_template_refs)};
+        // Don't track the dummy coinbase, because it can be modified in-place
+        // by submitSolution()
+        for (const CTransactionRef& tx : m_block_template->block.vtx | std::views::drop(1)) {
+            tx_refs[tx]++;
+        }
+    }
+
+    ~BlockTemplateImpl()
+    {
+        TxTemplateMap& tx_refs{*Assert(m_tx_template_refs)};
+        for (const CTransactionRef& tx : m_block_template->block.vtx | std::views::drop(1)) {
+            auto ref_count{tx_refs.find(tx)};
+            if (!Assume(ref_count != tx_refs.end())) break;
+            if (--ref_count->second == 0) {
+                tx_refs.erase(ref_count);
+            }
+        }
     }
 
     CBlockHeader getBlockHeader() override
@@ -918,7 +946,7 @@ class BlockTemplateImpl : public BlockTemplate
     std::unique_ptr<BlockTemplate> waitNext(BlockWaitOptions options) override
     {
         auto new_template = WaitAndCreateNewBlock(chainman(), notifications(), m_node.mempool.get(), m_block_template, options, m_assemble_options, m_interrupt_wait);
-        if (new_template) return std::make_unique<BlockTemplateImpl>(m_assemble_options, std::move(new_template), m_node);
+        if (new_template) return std::make_unique<BlockTemplateImpl>(m_assemble_options, std::move(new_template), m_node, m_tx_template_refs);
         return nullptr;
     }
 
@@ -935,12 +963,13 @@ class BlockTemplateImpl : public BlockTemplate
     ChainstateManager& chainman() { return *Assert(m_node.chainman); }
     KernelNotifications& notifications() { return *Assert(m_node.notifications); }
     NodeContext& m_node;
+    const std::shared_ptr<TxTemplateMap> m_tx_template_refs;
 };
 
 class MinerImpl : public Mining
 {
 public:
-    explicit MinerImpl(NodeContext& node) : m_node(node) {}
+    explicit MinerImpl(NodeContext& node) : m_node(node), m_tx_template_refs{std::make_shared<TxTemplateMap>()} {}
 
     bool isTestChain() override
     {
@@ -969,7 +998,14 @@ class MinerImpl : public Mining
 
         BlockAssembler::Options assemble_options{options};
         ApplyArgsManOptions(*Assert(m_node.args), assemble_options);
-        return std::make_unique<BlockTemplateImpl>(assemble_options, BlockAssembler{chainman().ActiveChainstate(), context()->mempool.get(), assemble_options}.CreateNewBlock(), m_node);
+        return std::make_unique<BlockTemplateImpl>(
+            assemble_options,
+            BlockAssembler{chainman().ActiveChainstate(),
+            context()->mempool.get(),
+            assemble_options}.CreateNewBlock(),
+            m_node,
+            m_tx_template_refs
+        );
     }
 
     bool checkBlock(const CBlock& block, const node::BlockCheckOptions& options, std::string& reason, std::string& debug) override
@@ -981,10 +1017,31 @@ class MinerImpl : public Mining
         return state.IsValid();
     }
 
+    MemoryLoad getMemoryLoad() override
+    {
+        CTxMemPool& mempool{*Assert(m_node.mempool)};
+        TxTemplateMap& tx_refs{*Assert(m_tx_template_refs)};
+        size_t usage_bytes{0};
+        {
+            LOG_TIME_MILLIS_WITH_CATEGORY("Calculate template transaction reference memory footprint", BCLog::BENCH);
+            for (const auto& [tx_ref, _] : tx_refs) {
+                if (!Assume(tx_ref)) continue;
+                if (mempool.exists(tx_ref->GetWitnessHash())) continue;
+                usage_bytes += RecursiveDynamicUsage(*tx_ref);
+            }
+        }
+        return {.usage = usage_bytes};;
+    }
+
     NodeContext* context() override { return &m_node; }
     ChainstateManager& chainman() { return *Assert(m_node.chainman); }
     KernelNotifications& notifications() { return *Assert(m_node.notifications); }
     NodeContext& m_node;
+
+    // Track how many templates (which we hold on to on behalf of connected
+    // clients) are referencing each transaction. This is used to track the
+    // memory footprint of non-mempool transactions.
+    std::shared_ptr<TxTemplateMap> m_tx_template_refs;
 };
 } // namespace
 } // namespace node

test/functional/interface_ipc.py

@@ -11,6 +11,7 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
+    assert_greater_than,
     assert_not_equal
 )
 from test_framework.wallet import MiniWallet
@@ -88,6 +89,12 @@ async def parse_and_deserialize_coinbase_tx(self, block_template, ctx):
         tx.deserialize(coinbase_data)
         return tx
 
+    async def destroy_template(self, template_obj, ctx):
+        """Destroy template if we received one."""
+        if template_obj is None or template_obj.to_dict() == {}:
+            return
+        await template_obj.result.destroy(ctx)
+
     def run_echo_test(self):
         self.log.info("Running echo test")
         async def async_routine():
@@ -188,6 +195,11 @@ async def async_routine():
             template7 = await template6.result.waitNext(ctx, waitoptions)
             assert_equal(template7.to_dict(), {})
 
+            self.log.debug("Memory load should be zero because there was no mempool churn")
+            with self.nodes[0].assert_debug_log(["Calculate template transaction reference memory footprint"]):
+                memory_load = await mining.result.getMemoryLoad(ctx)
+            assert_equal(memory_load.result.usage, 0)
+
             self.log.debug("interruptWait should abort the current wait")
             wait_started = asyncio.Event()
             async def wait_for_block():
@@ -212,9 +224,9 @@ async def interrupt_wait():
 
             current_block_height = self.nodes[0].getchaintips()[0]["height"]
             check_opts = self.capnp_modules['mining'].BlockCheckOptions()
-            template = await mining.result.createNewBlock(opts)
-            block = await self.parse_and_deserialize_block(template, ctx)
-            coinbase = await self.parse_and_deserialize_coinbase_tx(template, ctx)
+            template8 = await mining.result.createNewBlock(opts)
+            block = await self.parse_and_deserialize_block(template8, ctx)
+            coinbase = await self.parse_and_deserialize_coinbase_tx(template8, ctx)
             balance = miniwallet.get_balance()
             coinbase.vout[0].scriptPubKey = miniwallet.get_output_script()
             coinbase.vout[0].nValue = COIN
@@ -227,7 +239,7 @@ async def interrupt_wait():
             res = await mining.result.checkBlock(block.serialize(), check_opts)
             assert_equal(res.result, False)
             assert_equal(res.reason, "bad-version(0x00000000)")
-            res = await template.result.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())
+            res = await template8.result.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())
             assert_equal(res.result, False)
             self.log.debug("Submit a valid block")
             block.nVersion = original_version
@@ -238,21 +250,21 @@ async def interrupt_wait():
             assert_equal(res.result, True)
 
             # The remote template block will be mutated, capture the original:
-            remote_block_before = await self.parse_and_deserialize_block(template, ctx)
+            remote_block_before = await self.parse_and_deserialize_block(template8, ctx)
 
             self.log.debug("Submitted coinbase must include witness")
             assert_not_equal(coinbase.serialize_without_witness().hex(), coinbase.serialize().hex())
-            res = await template.result.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize_without_witness())
+            res = await template8.result.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize_without_witness())
             assert_equal(res.result, False)
 
             self.log.debug("Even a rejected submitBlock() mutates the template's block")
             # Can be used by clients to download and inspect the (rejected)
             # reconstructed block.
-            remote_block_after = await self.parse_and_deserialize_block(template, ctx)
+            remote_block_after = await self.parse_and_deserialize_block(template8, ctx)
             assert_not_equal(remote_block_before.serialize().hex(), remote_block_after.serialize().hex())
 
             self.log.debug("Submit again, with the witness")
-            res = await template.result.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())
+            res = await template8.result.submitSolution(ctx, block.nVersion, block.nTime, block.nNonce, coinbase.serialize())
             assert_equal(res.result, True)
 
             self.log.debug("Block should propagate")
@@ -270,14 +282,28 @@ async def interrupt_wait():
             assert_equal(res.result, False)
             assert_equal(res.reason, "inconclusive-not-best-prevblk")
 
+            self.log.debug("Reported memory load should be > 0")
+            # Clients are expected to drop references to stale block templates
+            # briefly after the tip updates. In practice we mainly care about
+            # the memory footprint caused by mempool churn, but this scenario
+            # is easier to test.
+            assert_greater_than((await mining.result.getMemoryLoad(ctx)).result.usage, 0)
+
             self.log.debug("Destroy template objects")
-            template.result.destroy(ctx)
-            template2.result.destroy(ctx)
-            template3.result.destroy(ctx)
-            template4.result.destroy(ctx)
-            template5.result.destroy(ctx)
-            template6.result.destroy(ctx)
-            template7.result.destroy(ctx)
+            await self.destroy_template(template8, ctx)
+            await self.destroy_template(template7, ctx)
+            await self.destroy_template(template6, ctx)
+            await self.destroy_template(template5, ctx)
+            await self.destroy_template(template4, ctx)
+            await self.destroy_template(template3, ctx)
+
+            # All templates with transactions have been released
+            self.log.debug("Reported memory load should be 0")
+            assert_equal((await mining.result.getMemoryLoad(ctx)).result.usage, 0)
+
+            await self.destroy_template(template2, ctx)
+            await self.destroy_template(template, ctx)
+
         asyncio.run(capnp.run(async_routine()))
 
     def run_test(self):