Add mempool statistics collector

src/Makefile.am

@@ -133,6 +133,8 @@ BITCOIN_CORE_H = \
   script/sign.h \
   script/standard.h \
   script/ismine.h \
+  stats/stats.h \
+  stats/stats_mempool.h \
   streams.h \
   support/allocators/secure.h \
   support/allocators/zeroafterfree.h \
@@ -205,6 +207,9 @@ libbitcoin_server_a_SOURCES = \
   rpc/server.cpp \
   script/sigcache.cpp \
   script/ismine.cpp \
+  stats/rpc_stats.cpp \
+  stats/stats.cpp \
+  stats/stats_mempool.cpp \
   timedata.cpp \
   torcontrol.cpp \
   txdb.cpp \

src/init.cpp

@@ -33,6 +33,7 @@
 #include "script/standard.h"
 #include "script/sigcache.h"
 #include "scheduler.h"
+#include "stats/stats.h"
 #include "timedata.h"
 #include "txdb.h"
 #include "txmempool.h"
@@ -509,6 +510,7 @@ std::string HelpMessage(HelpMessageMode mode)
         strUsage += HelpMessageOpt("-rpcservertimeout=<n>", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT));
     }
 
+    strUsage += CStats::getHelpString(showDebug);
     return strUsage;
 }
 
@@ -1134,6 +1136,10 @@ bool AppInitParameterInteraction()
             }
         }
     }
+
+    if (!CStats::parameterInteraction())
+        return false;
+
     return true;
 }
 
@@ -1669,5 +1675,7 @@ bool AppInitMain(boost::thread_group& threadGroup, CScheduler& scheduler)
         pwalletMain->postInitProcess(scheduler);
 #endif
 
+    CStats::DefaultStats()->startCollecting(scheduler);
+
     return !fRequestShutdown;
 }

src/memusage.h

@@ -6,6 +6,7 @@
 #define BITCOIN_MEMUSAGE_H
 
 #include "indirectmap.h"
+#include "prevector.h"
 
 #include <stdlib.h>
 

src/rpc/register.h

@@ -19,6 +19,8 @@ void RegisterMiscRPCCommands(CRPCTable &tableRPC);
 void RegisterMiningRPCCommands(CRPCTable &tableRPC);
 /** Register raw transaction RPC commands */
 void RegisterRawTransactionRPCCommands(CRPCTable &tableRPC);
+/** Register stats RPC commands */
+void RegisterStatsRPCCommands(CRPCTable &tableRPC);
 
 static inline void RegisterAllCoreRPCCommands(CRPCTable &t)
 {
@@ -27,6 +29,7 @@ static inline void RegisterAllCoreRPCCommands(CRPCTable &t)
     RegisterMiscRPCCommands(t);
     RegisterMiningRPCCommands(t);
     RegisterRawTransactionRPCCommands(t);
+    RegisterStatsRPCCommands(t);
 }
 
 #endif

src/stats/rpc_stats.cpp

@@ -0,0 +1,82 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "rpc/server.h"
+#include "stats/stats.h"
+#include "util.h"
+#include "utilstrencodings.h"
+
+#include <stdint.h>
+
+#include <univalue.h>
+
+UniValue getmempoolstats(const JSONRPCRequest& request)
+{
+    if (request.fHelp || request.params.size() != 0)
+        throw std::runtime_error(
+            "getmempoolstats\n"
+            "\nReturns the collected mempool statistics.\n"
+            "\nThe samples are segregated in multiple precision groups.\n"
+            "\nSamples time interval is not guaranteed to be constant, hence there\n"
+            "is a time delta in each sample relative to the last sample.\n"
+            "\nResult:\n"
+            "  [\n"
+            "    {\n"
+            "      \"sample_interval\"    : \"interval\",      (numeric) Interval target in seconds\n"
+            "      \"time_from\"          : \"timestamp\",     (numeric) Timestamp, first sample\n"
+            "      \"samples\"            : [\n"
+            "                                 [<delta_in_secs>,<tx_count>,<dynamic_mem_usage>,<min_fee_per_k>],\n"
+            "                                 [<delta_in_secs>,<tx_count>,<dynamic_mem_usage>,<min_fee_per_k>],\n"
+            "                                 ...\n"
+            "                             ]\n"
+            "    }\n"
+            "    ,...\n"
+            "  ]\n"
+            "\nExamples:\n" +
+            HelpExampleCli("getmempoolstats", "") + HelpExampleRpc("getmempoolstats", ""));
+
+    // get stats from the core stats model
+    uint64_t timeFrom = 0;
+    std::vector<unsigned int> groups = CStats::DefaultStats()->mempoolCollector->getPrecisionGroupsAndIntervals();
+
+    UniValue groupsUni(UniValue::VARR);
+    for (unsigned int i = 0; i < groups.size(); i++) {
+
+        MempoolSamplesVector samples = CStats::DefaultStats()->mempoolCollector->getSamplesForPrecision(i, timeFrom);
+
+        // use "flat" json encoding for performance reasons
+        UniValue samplesObj(UniValue::VARR);
+        for (const struct CStatsMempoolSample& sample : samples) {
+            UniValue singleSample(UniValue::VARR);
+            singleSample.push_back(UniValue((uint64_t)sample.timeDelta));
+            singleSample.push_back(UniValue((uint64_t)sample.txCount));
+            singleSample.push_back(UniValue((uint64_t)sample.dynMemUsage));
+            singleSample.push_back(UniValue(sample.minFeePerK));
+            samplesObj.push_back(singleSample);
+        }
+
+        UniValue sampleGroup(UniValue::VOBJ);
+        sampleGroup.push_back(Pair("sample_interval", (int)groups[i]));
+        sampleGroup.push_back(Pair("time_from", timeFrom));
+        sampleGroup.push_back(Pair("samples", samplesObj));
+
+        groupsUni.push_back(sampleGroup);
+    }
+
+    return groupsUni;
+}
+
+static const CRPCCommand commands[] =
+{
+    //  category              name                      actor (function)         okSafe argNames
+    //  --------------------- ------------------------  -----------------------  ------ ----------
+        {"stats",             "getmempoolstats",        &getmempoolstats,        true,  {}},
+
+};
+
+void RegisterStatsRPCCommands(CRPCTable &t)
+{
+    for (unsigned int vcidx = 0; vcidx < ARRAYLEN(commands); vcidx++)
+        t.appendCommand(commands[vcidx].name, &commands[vcidx]);
+}

src/stats/stats.cpp

@@ -0,0 +1,81 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "stats/stats.h"
+
+#include "memusage.h"
+#include "utiltime.h"
+
+#include "util.h"
+
+
+const size_t DEFAULT_MAX_STATS_MEMORY = 10 * 1024 * 1024; // 10 MB
+const bool DEFAULT_STATISTICS_ENABLED = false;
+const static unsigned int STATS_COLLECT_INTERVAL = 2000; // 2 secs
+
+CStats* CStats::sharedInstance = NULL;
+
+CStats* CStats::DefaultStats()
+{
+    if (!sharedInstance)
+        sharedInstance = new CStats();
+
+    return sharedInstance;
+}
+
+CStats::CStats() : statsEnabled(false), maxStatsMemory(0)
+{
+    /* initialize the mempool stats collector */
+    mempoolCollector = std::unique_ptr<CStatsMempool>(new CStatsMempool(STATS_COLLECT_INTERVAL));
+}
+
+CStats::~CStats()
+{
+
+}
+
+std::string CStats::getHelpString(bool showDebug)
+{
+    std::string strUsage = HelpMessageGroup(_("Statistic options:"));
+    strUsage += HelpMessageOpt("-statsenable=", strprintf("Enable statistics (default: %u)", DEFAULT_STATISTICS_ENABLED));
+    strUsage += HelpMessageOpt("-statsmaxmemorytarget=<n>", strprintf(_("Set the memory limit target for statistics in bytes (default: %u)"), DEFAULT_MAX_STATS_MEMORY));
+
+    return strUsage;
+}
+
+bool CStats::parameterInteraction()
+{
+    if (GetBoolArg("-statsenable", DEFAULT_STATISTICS_ENABLED))
+        DefaultStats()->setMaxMemoryUsageTarget(GetArg("-statsmaxmemorytarget", DEFAULT_MAX_STATS_MEMORY));
+
+    return true;
+}
+
+void CStats::startCollecting(CScheduler& scheduler)
+{
+    if (statsEnabled) {
+        // dispatch the scheduler task
+        scheduler.scheduleEvery(std::bind(&CStats::collectCallback, this), STATS_COLLECT_INTERVAL);
+    }
+}
+
+void CStats::collectCallback()
+{
+    if (!statsEnabled)
+        return;
+
+    if (mempoolCollector->addMempoolSamples(maxStatsMemory)) {
+        // fire the signal if the stats did change
+        MempoolStatsDidChange();
+    }
+}
+
+void CStats::setMaxMemoryUsageTarget(size_t maxMem)
+{
+    statsEnabled = (maxMem > 0);
+    maxStatsMemory = maxMem;
+
+    // for now: give 100% to mempool stats
+    mempoolCollector->setMaxMemoryUsageTarget(maxMem);
+}

src/stats/stats.h

@@ -0,0 +1,61 @@
+// Copyright (c) 2016 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_STATS_H
+#define BITCOIN_STATS_H
+
+#include "amount.h"
+#include "scheduler.h"
+#include "stats/stats_mempool.h"
+
+#include <atomic>
+#include <stdlib.h>
+
+#include <boost/signals2/signal.hpp>
+
+// Class that manages various types of statistics and its memory consumption
+class CStats
+{
+private:
+    static CStats* sharedInstance; //!< singleton instance
+    std::atomic<bool> statsEnabled;
+
+    //maximum amount of memory to use for the stats
+    std::atomic<size_t> maxStatsMemory;
+
+    /* collect callback through the scheduler task */
+    void collectCallback();
+
+public:
+
+    static CStats* DefaultStats(); //shared instance
+
+    CStats();
+    ~CStats();
+
+    /* get the statistics module help strings */
+    static std::string getHelpString(bool showDebug);
+
+    /* access the parameters and map it to the internal model */
+    static bool parameterInteraction();
+
+    /* dispatch the stats collector repetitive scheduler task */
+    void startCollecting(CScheduler& scheduler);
+
+    /* set the target for the maximum memory consumption (in bytes) */
+    void setMaxMemoryUsageTarget(size_t maxMem);
+
+
+
+    /* COLLECTOR INSTANCES
+     * =================== */
+    std::unique_ptr<CStatsMempool> mempoolCollector;
+
+
+    /* SIGNALS
+     * ======= */
+    boost::signals2::signal<void(void)> MempoolStatsDidChange; //mempool signal
+};
+
+#endif // BITCOIN_STATS_H