Merge bitcoin#17292: Add new mempool benchmarks for a complex pool

b0c774b Add new mempool benchmarks for a complex pool (Jeremy Rubin)

Pull request description:

  This PR is related to bitcoin#17268.

  It adds a mempool stress test which makes a really big complicated tx graph, and then, similar to mempool_eviction test, trims the size.

  The test setup is to make 100 original transactions with Rand(10)+2 outputs each.

  Then, 800 times:

  we create a new transaction with Rand(10) + 1 parents that are randomly sampled from all existing transactions (with unspent outputs). From each such parent, we then select Rand(remaining outputs) +1 50% of the time, or 1 outputs 50% of the time.

  Then, we trim the size to 3/4. Then we trim it to just a single transaction.

  This creates, hopefully, a big bundle of transactions with lots of complex structure, that should really put a strain on the mempool graph algorithms.

  This ends up testing both the descendant and ancestor tracking.

  I don't love that the test is "unstable". That is, in order to compare this test to another, you really can't modify any of the internal state because it will have a different order of invocations of the deterministic randomness. However, it certainly suffices for comparing branches.

Top commit has no ACKs.

Tree-SHA512: cabe96b849b9885878e20eec558915e921d49e6ed1e4b011b22ca191b4c99aa28930a8b963784c9adf78cc8b034a655513f7a0da865e280a1214ae15ebb1d574

src/Makefile.bench.include

@@ -30,6 +30,7 @@ bench_bench_syscoin_SOURCES = \
   bench/gcs_filter.cpp \
   bench/merkle_root.cpp \
   bench/mempool_eviction.cpp \
+  bench/mempool_stress.cpp \
   bench/rpc_blockchain.cpp \
   bench/rpc_mempool.cpp \
   bench/util_time.cpp \

src/bench/mempool_stress.cpp

@@ -0,0 +1,93 @@
+// Copyright (c) 2011-2019 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <bench/bench.h>
+#include <policy/policy.h>
+#include <txmempool.h>
+
+#include <vector>
+
+static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs)
+{
+    int64_t nTime = 0;
+    unsigned int nHeight = 1;
+    bool spendsCoinbase = false;
+    unsigned int sigOpCost = 4;
+    LockPoints lp;
+    pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, spendsCoinbase, sigOpCost, lp));
+}
+
+struct Available {
+    CTransactionRef ref;
+    size_t vin_left{0};
+    size_t tx_count;
+    Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){}
+    Available& operator=(Available other) {
+        ref = other.ref;
+        vin_left = other.vin_left;
+        tx_count = other.tx_count;
+        return *this;
+    }
+};
+
+static void ComplexMemPool(benchmark::State& state)
+{
+    FastRandomContext det_rand{true};
+    std::vector<Available> available_coins;
+    std::vector<CTransactionRef> ordered_coins;
+    // Create some base transactions
+    size_t tx_counter = 1;
+    for (auto x = 0; x < 100; ++x) {
+        CMutableTransaction tx = CMutableTransaction();
+        tx.vin.resize(1);
+        tx.vin[0].scriptSig = CScript() << CScriptNum(tx_counter);
+        tx.vin[0].scriptWitness.stack.push_back(CScriptNum(x).getvch());
+        tx.vout.resize(det_rand.randrange(10)+2);
+        for (auto& out : tx.vout) {
+            out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL;
+            out.nValue = 10 * COIN;
+        }
+        ordered_coins.emplace_back(MakeTransactionRef(tx));
+        available_coins.emplace_back(ordered_coins.back(), tx_counter++);
+    }
+    for (auto x = 0; x < 800 && !available_coins.empty(); ++x) {
+        CMutableTransaction tx = CMutableTransaction();
+        size_t n_ancestors = det_rand.randrange(10)+1;
+        for (size_t ancestor = 0; ancestor < n_ancestors && !available_coins.empty(); ++ancestor){
+            size_t idx = det_rand.randrange(available_coins.size());
+            Available coin = available_coins[idx];
+            uint256 hash = coin.ref->GetHash();
+            // biased towards taking just one ancestor, but maybe more
+            size_t n_to_take = det_rand.randrange(2) == 0 ? 1 : 1+det_rand.randrange(coin.ref->vout.size() - coin.vin_left);
+            for (size_t i = 0; i < n_to_take; ++i) {
+                tx.vin.emplace_back();
+                tx.vin.back().prevout = COutPoint(hash, coin.vin_left++);
+                tx.vin.back().scriptSig = CScript() << coin.tx_count;
+                tx.vin.back().scriptWitness.stack.push_back(CScriptNum(coin.tx_count).getvch());
+            }
+            if  (coin.vin_left == coin.ref->vin.size()) {
+                coin = available_coins.back();
+                available_coins.pop_back();
+            }
+            tx.vout.resize(det_rand.randrange(10)+2);
+            for (auto& out : tx.vout) {
+                out.scriptPubKey = CScript() << CScriptNum(tx_counter) << OP_EQUAL;
+                out.nValue = 10 * COIN;
+            }
+        }
+        ordered_coins.emplace_back(MakeTransactionRef(tx));
+        available_coins.emplace_back(ordered_coins.back(), tx_counter++);
+    }
+    CTxMemPool pool;
+    LOCK2(cs_main, pool.cs);
+    while (state.KeepRunning()) {
+        for (auto& tx : ordered_coins) {
+            AddTx(tx, pool);
+        }
+        pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4);
+        pool.TrimToSize(GetVirtualTransactionSize(*ordered_coins.front()));
+    }
+}
+
+BENCHMARK(ComplexMemPool, 1);