Permalink
Browse files

Track transaction packages in CTxMemPoolEntry

Associate with each CTxMemPoolEntry all the size/fees of descendant
mempool transactions.  Sort mempool by max(feerate of entry, feerate
of descendants).  Update statistics on-the-fly as transactions enter
or leave the mempool.

Also add ancestor and descendant limiting, so that transactions can
be rejected if the number or size of unconfirmed ancestors exceeds
a target, or if adding a transaction would cause some other mempool
entry to have too many (or too large) a set of unconfirmed in-
mempool descendants.
  • Loading branch information...
sdaftuar committed Jul 15, 2015
1 parent 34628a1 commit 5add7a74a672cb12b0a2a630d318d9bc64dd0f77
Showing with 993 additions and 65 deletions.
  1. +1 −0 qa/pull-tester/rpc-tests.sh
  2. +107 −0 qa/rpc-tests/mempool_packages.py
  3. +4 −0 src/init.cpp
  4. +24 −3 src/main.cpp
  5. +8 −0 src/main.h
  6. +15 −3 src/memusage.h
  7. +6 −0 src/rpcblockchain.cpp
  8. +139 −10 src/test/mempool_tests.cpp
  9. +423 −42 src/txmempool.cpp
  10. +266 −7 src/txmempool.h
@@ -57,6 +57,7 @@ testScriptsExt=(
'invalidblockrequest.py'
# 'forknotify.py'
'p2p-acceptblock.py'
'mempool_packages.py'
);

#if [ "x$ENABLE_ZMQ" = "x1" ]; then
@@ -0,0 +1,107 @@
#!/usr/bin/env python2
# Copyright (c) 2014-2015 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.

# Test descendant package tracking code

from test_framework.test_framework import BitcoinTestFramework
from test_framework.util import *

def satoshi_round(amount):
return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)

class MempoolPackagesTest(BitcoinTestFramework):

def setup_network(self):
self.nodes = []
self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-relaypriority=0"]))
self.is_network_split = False
self.sync_all()

# Build a transaction that spends parent_txid:vout
# Return amount sent
def chain_transaction(self, parent_txid, vout, value, fee, num_outputs):
send_value = satoshi_round((value - fee)/num_outputs)
inputs = [ {'txid' : parent_txid, 'vout' : vout} ]
outputs = {}
for i in xrange(num_outputs):
outputs[self.nodes[0].getnewaddress()] = send_value
rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
signedtx = self.nodes[0].signrawtransaction(rawtx)
txid = self.nodes[0].sendrawtransaction(signedtx['hex'])
fulltx = self.nodes[0].getrawtransaction(txid, 1)
assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output
return (txid, send_value)

def run_test(self):
''' Mine some blocks and have them mature. '''
self.nodes[0].generate(101)
utxo = self.nodes[0].listunspent(10)
txid = utxo[0]['txid']
vout = utxo[0]['vout']
value = utxo[0]['amount']

fee = Decimal("0.0001")
# 100 transactions off a confirmed tx should be fine
chain = []
for i in xrange(100):
(txid, sent_value) = self.chain_transaction(txid, 0, value, fee, 1)
value = sent_value
chain.append(txid)

# Check mempool has 100 transactions in it, and descendant
# count and fees should look correct
mempool = self.nodes[0].getrawmempool(True)
assert_equal(len(mempool), 100)
descendant_count = 1
descendant_fees = 0
descendant_size = 0
SATOSHIS = 100000000

for x in reversed(chain):
assert_equal(mempool[x]['descendantcount'], descendant_count)
descendant_fees += mempool[x]['fee']
assert_equal(mempool[x]['descendantfees'], SATOSHIS*descendant_fees)
descendant_size += mempool[x]['size']
assert_equal(mempool[x]['descendantsize'], descendant_size)
descendant_count += 1

# Adding one more transaction on to the chain should fail.
try:
self.chain_transaction(txid, vout, value, fee, 1)
except JSONRPCException as e:
print "too-long-ancestor-chain successfully rejected"

# TODO: test ancestor size limits

# Now test descendant chain limits
txid = utxo[1]['txid']
value = utxo[1]['amount']
vout = utxo[1]['vout']

transaction_package = []
# First create one parent tx with 10 children
(txid, sent_value) = self.chain_transaction(txid, vout, value, fee, 10)
parent_transaction = txid
for i in xrange(10):
transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value})

for i in xrange(1000):
utxo = transaction_package.pop(0)
try:
(txid, sent_value) = self.chain_transaction(utxo['txid'], utxo['vout'], utxo['amount'], fee, 10)
for j in xrange(10):
transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value})
if i == 998:
mempool = self.nodes[0].getrawmempool(True)
assert_equal(mempool[parent_transaction]['descendantcount'], 1000)
except JSONRPCException as e:
print e.error['message']
assert_equal(i, 999)
print "tx that would create too large descendant package successfully rejected"

# TODO: test descendant size limits

if __name__ == '__main__':
MempoolPackagesTest().main()
@@ -411,6 +411,10 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages");
strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1));
strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0));
strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT));
strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT));
strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
}
string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http"; // Don't translate these and qt below
if (mode == HMM_BITCOIN_QT)
@@ -921,6 +921,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
REJECT_HIGHFEE, "absurdly-high-fee",
strprintf("%d > %d", nFees, ::minRelayTxFee.GetFee(nSize) * 10000));

// Calculate in-mempool ancestors, up to a limit.
CTxMemPool::setEntries setAncestors;
size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT);
size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000;
size_t nLimitDescendants = GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT);
size_t nLimitDescendantSize = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000;
std::string errString;
if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) {
return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString);
}

// Check against previous transactions
// This is done last to help prevent CPU exhaustion denial-of-service attacks.
if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true))
@@ -942,7 +953,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
}

// Store transaction in memory
pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload());
}

SyncWithWallets(tx, NULL);
@@ -2033,13 +2044,23 @@ bool static DisconnectTip(CValidationState &state) {
if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED))
return false;
// Resurrect mempool transactions from the disconnected block.
std::vector<uint256> vHashUpdate;
BOOST_FOREACH(const CTransaction &tx, block.vtx) {
// ignore validation errors in resurrected transactions
list<CTransaction> removed;
CValidationState stateDummy;
if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL))
if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) {
mempool.remove(tx, removed, true);
} else if (mempool.exists(tx.GetHash())) {
vHashUpdate.push_back(tx.GetHash());
}
}
// AcceptToMemoryPool/addUnchecked all assume that new mempool entries have
// no in-mempool children, which is generally not true when adding
// previously-confirmed transactions back to the mempool.
// UpdateTransactionsFromBlock finds descendants of any transactions in this
// block that were added back and cleans up the mempool state.
mempool.UpdateTransactionsFromBlock(vHashUpdate);
mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight);
mempool.check(pcoinsTip);
// Update chainActive and related variables.
@@ -4258,7 +4279,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
LogPrint("mempool", "AcceptToMemoryPool: peer=%d %s: accepted %s (poolsz %u)\n",
pfrom->id, pfrom->cleanSubVer,
tx.GetHash().ToString(),
mempool.mapTx.size());
mempool.size());

// Recursively process any orphan transactions that depended on this one
set<NodeId> setMisbehaving;
@@ -43,6 +43,14 @@ struct CNodeStateStats;
static const bool DEFAULT_ALERTS = true;
/** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
/** Default for -limitancestorcount, max number of in-mempool ancestors */
static const unsigned int DEFAULT_ANCESTOR_LIMIT = 100;
/** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */
static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 900;
/** Default for -limitdescendantcount, max number of in-mempool descendants */
static const unsigned int DEFAULT_DESCENDANT_LIMIT = 1000;
/** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */
static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 2500;
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
@@ -74,18 +74,30 @@ static inline size_t DynamicUsage(const std::vector<X>& v)
return MallocUsage(v.capacity() * sizeof(X));
}

template<typename X>
static inline size_t DynamicUsage(const std::set<X>& s)
template<typename X, typename Y>
static inline size_t DynamicUsage(const std::set<X, Y>& s)
{
return MallocUsage(sizeof(stl_tree_node<X>)) * s.size();
}

template<typename X, typename Y>
static inline size_t DynamicUsage(const std::map<X, Y>& m)
static inline size_t IncrementalDynamicUsage(const std::set<X, Y>& s)
{
return MallocUsage(sizeof(stl_tree_node<X>));
}

template<typename X, typename Y, typename Z>
static inline size_t DynamicUsage(const std::map<X, Y, Z>& m)
{
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size();
}

template<typename X, typename Y, typename Z>
static inline size_t IncrementalDynamicUsage(const std::map<X, Y, Z>& m)
{
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >));
}

// Boost data structures

template<typename X>
@@ -191,6 +191,9 @@ UniValue mempoolToJSON(bool fVerbose = false)
info.push_back(Pair("height", (int)e.GetHeight()));
info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight())));
info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height())));
info.push_back(Pair("descendantcount", e.GetCountWithDescendants()));
info.push_back(Pair("descendantsize", e.GetSizeWithDescendants()));
info.push_back(Pair("descendantfees", e.GetFeesWithDescendants()));
const CTransaction& tx = e.GetTx();
set<string> setDepends;
BOOST_FOREACH(const CTxIn& txin, tx.vin)
@@ -245,6 +248,9 @@ UniValue getrawmempool(const UniValue& params, bool fHelp)
" \"height\" : n, (numeric) block height when transaction entered pool\n"
" \"startingpriority\" : n, (numeric) priority when transaction entered pool\n"
" \"currentpriority\" : n, (numeric) transaction priority now\n"
" \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n"
" \"descendantsize\" : n, (numeric) size of in-mempool descendants (including this one)\n"
" \"descendantfees\" : n, (numeric) fees of in-mempool descendants (including this one)\n"
" \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n"
" \"transactionid\", (string) parent transaction id\n"
" ... ]\n"
Oops, something went wrong.

0 comments on commit 5add7a7

Please sign in to comment.