Track transaction packages in CTxMemPoolEntry

Associate with each CTxMemPoolEntry all the size/fees of descendant
mempool transactions.  Sort mempool by max(feerate of entry, feerate
of descendants).  Update statistics on-the-fly as transactions enter
or leave the mempool.

Also add ancestor and descendant limiting, so that transactions can
be rejected if the number or size of unconfirmed ancestors exceeds
a target, or if adding a transaction would cause some other mempool
entry to have too many (or too large) a set of unconfirmed in-
mempool descendants.

qa/pull-tester/rpc-tests.sh

@@ -57,6 +57,7 @@ testScriptsExt=(
     'invalidblockrequest.py'
 #    'forknotify.py'
     'p2p-acceptblock.py'
+    'mempool_packages.py'
 );
 
 #if [ "x$ENABLE_ZMQ" = "x1" ]; then

qa/rpc-tests/mempool_packages.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python2
+# Copyright (c) 2014-2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Test descendant package tracking code
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+
+def satoshi_round(amount):
+    return  Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)
+
+class MempoolPackagesTest(BitcoinTestFramework):
+
+    def setup_network(self):
+        self.nodes = []
+        self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-relaypriority=0"]))
+        self.is_network_split = False
+        self.sync_all()
+
+    # Build a transaction that spends parent_txid:vout
+    # Return amount sent
+    def chain_transaction(self, parent_txid, vout, value, fee, num_outputs):
+        send_value = satoshi_round((value - fee)/num_outputs)
+        inputs = [ {'txid' : parent_txid, 'vout' : vout} ]
+        outputs = {}
+        for i in xrange(num_outputs):
+            outputs[self.nodes[0].getnewaddress()] = send_value
+        rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
+        signedtx = self.nodes[0].signrawtransaction(rawtx)
+        txid = self.nodes[0].sendrawtransaction(signedtx['hex'])
+        fulltx = self.nodes[0].getrawtransaction(txid, 1)
+        assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output
+        return (txid, send_value)
+
+    def run_test(self):
+        ''' Mine some blocks and have them mature. '''
+        self.nodes[0].generate(101)
+        utxo = self.nodes[0].listunspent(10)
+        txid = utxo[0]['txid']
+        vout = utxo[0]['vout']
+        value = utxo[0]['amount']
+
+        fee = Decimal("0.0001")
+        # 100 transactions off a confirmed tx should be fine
+        chain = []
+        for i in xrange(100):
+            (txid, sent_value) = self.chain_transaction(txid, 0, value, fee, 1)
+            value = sent_value
+            chain.append(txid)
+
+        # Check mempool has 100 transactions in it, and descendant
+        # count and fees should look correct
+        mempool = self.nodes[0].getrawmempool(True)
+        assert_equal(len(mempool), 100)
+        descendant_count = 1
+        descendant_fees = 0
+        descendant_size = 0
+        SATOSHIS = 100000000
+
+        for x in reversed(chain):
+            assert_equal(mempool[x]['descendantcount'], descendant_count)
+            descendant_fees += mempool[x]['fee']
+            assert_equal(mempool[x]['descendantfees'], SATOSHIS*descendant_fees)
+            descendant_size += mempool[x]['size']
+            assert_equal(mempool[x]['descendantsize'], descendant_size)
+            descendant_count += 1
+
+        # Adding one more transaction on to the chain should fail.
+        try:
+            self.chain_transaction(txid, vout, value, fee, 1)
+        except JSONRPCException as e:
+            print "too-long-ancestor-chain successfully rejected"
+
+        # TODO: test ancestor size limits
+
+        # Now test descendant chain limits
+        txid = utxo[1]['txid']
+        value = utxo[1]['amount']
+        vout = utxo[1]['vout']
+
+        transaction_package = []
+        # First create one parent tx with 10 children
+        (txid, sent_value) = self.chain_transaction(txid, vout, value, fee, 10)
+        parent_transaction = txid
+        for i in xrange(10):
+            transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value})
+
+        for i in xrange(1000):
+            utxo = transaction_package.pop(0)
+            try:
+                (txid, sent_value) = self.chain_transaction(utxo['txid'], utxo['vout'], utxo['amount'], fee, 10)
+                for j in xrange(10):
+                    transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value})
+                if i == 998:
+                    mempool = self.nodes[0].getrawmempool(True)
+                    assert_equal(mempool[parent_transaction]['descendantcount'], 1000)
+            except JSONRPCException as e:
+                print e.error['message']
+                assert_equal(i, 999)
+                print "tx that would create too large descendant package successfully rejected"
+
+        # TODO: test descendant size limits
+
+if __name__ == '__main__':
+    MempoolPackagesTest().main()

src/init.cpp

@@ -411,6 +411,10 @@ std::string HelpMessage(HelpMessageMode mode)
         strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages");
         strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1));
         strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0));
+        strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT));
+        strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT));
+        strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
+        strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
     }
     string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http"; // Don't translate these and qt below
     if (mode == HMM_BITCOIN_QT)

src/main.cpp

@@ -921,6 +921,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
                 REJECT_HIGHFEE, "absurdly-high-fee",
                 strprintf("%d > %d", nFees, ::minRelayTxFee.GetFee(nSize) * 10000));
 
+        // Calculate in-mempool ancestors, up to a limit.
+        CTxMemPool::setEntries setAncestors;
+        size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT);
+        size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000;
+        size_t nLimitDescendants = GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT);
+        size_t nLimitDescendantSize = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000;
+        std::string errString;
+        if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) {
+            return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString);
+        }
+
         // Check against previous transactions
         // This is done last to help prevent CPU exhaustion denial-of-service attacks.
         if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true))
@@ -942,7 +953,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
         }
 
         // Store transaction in memory
-        pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
+        pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload());
     }
 
     SyncWithWallets(tx, NULL);
@@ -2033,13 +2044,23 @@ bool static DisconnectTip(CValidationState &state) {
     if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED))
         return false;
     // Resurrect mempool transactions from the disconnected block.
+    std::vector<uint256> vHashUpdate;
     BOOST_FOREACH(const CTransaction &tx, block.vtx) {
         // ignore validation errors in resurrected transactions
         list<CTransaction> removed;
         CValidationState stateDummy;
-        if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL))
+        if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) {
             mempool.remove(tx, removed, true);
+        } else if (mempool.exists(tx.GetHash())) {
+            vHashUpdate.push_back(tx.GetHash());
+        }
     }
+    // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have
+    // no in-mempool children, which is generally not true when adding
+    // previously-confirmed transactions back to the mempool.
+    // UpdateTransactionsFromBlock finds descendants of any transactions in this
+    // block that were added back and cleans up the mempool state.
+    mempool.UpdateTransactionsFromBlock(vHashUpdate);
     mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight);
     mempool.check(pcoinsTip);
     // Update chainActive and related variables.
@@ -4258,7 +4279,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
             LogPrint("mempool", "AcceptToMemoryPool: peer=%d %s: accepted %s (poolsz %u)\n",
                 pfrom->id, pfrom->cleanSubVer,
                 tx.GetHash().ToString(),
-                mempool.mapTx.size());
+                mempool.size());
 
             // Recursively process any orphan transactions that depended on this one
             set<NodeId> setMisbehaving;

src/main.h

@@ -43,6 +43,14 @@ struct CNodeStateStats;
 static const bool DEFAULT_ALERTS = true;
 /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
 static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
+/** Default for -limitancestorcount, max number of in-mempool ancestors */
+static const unsigned int DEFAULT_ANCESTOR_LIMIT = 100;
+/** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */
+static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 900;
+/** Default for -limitdescendantcount, max number of in-mempool descendants */
+static const unsigned int DEFAULT_DESCENDANT_LIMIT = 1000;
+/** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */
+static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 2500;
 /** The maximum size of a blk?????.dat file (since 0.8) */
 static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
 /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */

src/memusage.h

@@ -74,18 +74,30 @@ static inline size_t DynamicUsage(const std::vector<X>& v)
     return MallocUsage(v.capacity() * sizeof(X));
 }
 
-template<typename X>
-static inline size_t DynamicUsage(const std::set<X>& s)
+template<typename X, typename Y>
+static inline size_t DynamicUsage(const std::set<X, Y>& s)
 {
     return MallocUsage(sizeof(stl_tree_node<X>)) * s.size();
 }
 
 template<typename X, typename Y>
-static inline size_t DynamicUsage(const std::map<X, Y>& m)
+static inline size_t IncrementalDynamicUsage(const std::set<X, Y>& s)
+{
+    return MallocUsage(sizeof(stl_tree_node<X>));
+}
+
+template<typename X, typename Y, typename Z>
+static inline size_t DynamicUsage(const std::map<X, Y, Z>& m)
 {
     return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size();
 }
 
+template<typename X, typename Y, typename Z>
+static inline size_t IncrementalDynamicUsage(const std::map<X, Y, Z>& m)
+{
+    return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >));
+}
+
 // Boost data structures
 
 template<typename X>

src/rpcblockchain.cpp

@@ -191,6 +191,9 @@ UniValue mempoolToJSON(bool fVerbose = false)
             info.push_back(Pair("height", (int)e.GetHeight()));
             info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight())));
             info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height())));
+            info.push_back(Pair("descendantcount", e.GetCountWithDescendants()));
+            info.push_back(Pair("descendantsize", e.GetSizeWithDescendants()));
+            info.push_back(Pair("descendantfees", e.GetFeesWithDescendants()));
             const CTransaction& tx = e.GetTx();
             set<string> setDepends;
             BOOST_FOREACH(const CTxIn& txin, tx.vin)
@@ -245,6 +248,9 @@ UniValue getrawmempool(const UniValue& params, bool fHelp)
             "    \"height\" : n,           (numeric) block height when transaction entered pool\n"
             "    \"startingpriority\" : n, (numeric) priority when transaction entered pool\n"
             "    \"currentpriority\" : n,  (numeric) transaction priority now\n"
+            "    \"descendantcount\" : n,  (numeric) number of in-mempool descendant transactions (including this one)\n"
+            "    \"descendantsize\" : n,   (numeric) size of in-mempool descendants (including this one)\n"
+            "    \"descendantfees\" : n,   (numeric) fees of in-mempool descendants (including this one)\n"
             "    \"depends\" : [           (array) unconfirmed transactions used as inputs for this transaction\n"
             "        \"transactionid\",    (string) parent transaction id\n"
             "       ... ]\n"