Permalink
Browse files

Merge pull request #6654

5add7a7 Track transaction packages in CTxMemPoolEntry (Suhas Daftuar)
34628a1 TxMemPool: Change mapTx to a boost::multi_index_container (Ashley Holman)
  • Loading branch information...
laanwj committed Sep 21, 2015
2 parents d5d1d2e + 5add7a7 commit b0ce4508b04639189a6d0567782e27eeee5f73d6
Showing with 1,098 additions and 75 deletions.
  1. +1 −0 qa/pull-tester/rpc-tests.sh
  2. +107 −0 qa/rpc-tests/mempool_packages.py
  3. +4 −0 src/init.cpp
  4. +24 −3 src/main.cpp
  5. +8 −0 src/main.h
  6. +15 −3 src/memusage.h
  7. +4 −4 src/miner.cpp
  8. +8 −3 src/rpcblockchain.cpp
  9. +181 −0 src/test/mempool_tests.cpp
  10. +443 −58 src/txmempool.cpp
  11. +303 −4 src/txmempool.h
@@ -57,6 +57,7 @@ testScriptsExt=(
'invalidblockrequest.py'
# 'forknotify.py'
'p2p-acceptblock.py'
+ 'mempool_packages.py'
);
#if [ "x$ENABLE_ZMQ" = "x1" ]; then
@@ -0,0 +1,107 @@
+#!/usr/bin/env python2
+# Copyright (c) 2014-2015 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Test descendant package tracking code
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import *
+
+def satoshi_round(amount):
+ return Decimal(amount).quantize(Decimal('0.00000001'), rounding=ROUND_DOWN)
+
+class MempoolPackagesTest(BitcoinTestFramework):
+
+ def setup_network(self):
+ self.nodes = []
+ self.nodes.append(start_node(0, self.options.tmpdir, ["-maxorphantx=1000", "-relaypriority=0"]))
+ self.is_network_split = False
+ self.sync_all()
+
+ # Build a transaction that spends parent_txid:vout
+ # Return amount sent
+ def chain_transaction(self, parent_txid, vout, value, fee, num_outputs):
+ send_value = satoshi_round((value - fee)/num_outputs)
+ inputs = [ {'txid' : parent_txid, 'vout' : vout} ]
+ outputs = {}
+ for i in xrange(num_outputs):
+ outputs[self.nodes[0].getnewaddress()] = send_value
+ rawtx = self.nodes[0].createrawtransaction(inputs, outputs)
+ signedtx = self.nodes[0].signrawtransaction(rawtx)
+ txid = self.nodes[0].sendrawtransaction(signedtx['hex'])
+ fulltx = self.nodes[0].getrawtransaction(txid, 1)
+ assert(len(fulltx['vout']) == num_outputs) # make sure we didn't generate a change output
+ return (txid, send_value)
+
+ def run_test(self):
+ ''' Mine some blocks and have them mature. '''
+ self.nodes[0].generate(101)
+ utxo = self.nodes[0].listunspent(10)
+ txid = utxo[0]['txid']
+ vout = utxo[0]['vout']
+ value = utxo[0]['amount']
+
+ fee = Decimal("0.0001")
+ # 100 transactions off a confirmed tx should be fine
+ chain = []
+ for i in xrange(100):
+ (txid, sent_value) = self.chain_transaction(txid, 0, value, fee, 1)
+ value = sent_value
+ chain.append(txid)
+
+ # Check mempool has 100 transactions in it, and descendant
+ # count and fees should look correct
+ mempool = self.nodes[0].getrawmempool(True)
+ assert_equal(len(mempool), 100)
+ descendant_count = 1
+ descendant_fees = 0
+ descendant_size = 0
+ SATOSHIS = 100000000
+
+ for x in reversed(chain):
+ assert_equal(mempool[x]['descendantcount'], descendant_count)
+ descendant_fees += mempool[x]['fee']
+ assert_equal(mempool[x]['descendantfees'], SATOSHIS*descendant_fees)
+ descendant_size += mempool[x]['size']
+ assert_equal(mempool[x]['descendantsize'], descendant_size)
+ descendant_count += 1
+
+ # Adding one more transaction on to the chain should fail.
+ try:
+ self.chain_transaction(txid, vout, value, fee, 1)
+ except JSONRPCException as e:
+ print "too-long-ancestor-chain successfully rejected"
+
+ # TODO: test ancestor size limits
+
+ # Now test descendant chain limits
+ txid = utxo[1]['txid']
+ value = utxo[1]['amount']
+ vout = utxo[1]['vout']
+
+ transaction_package = []
+ # First create one parent tx with 10 children
+ (txid, sent_value) = self.chain_transaction(txid, vout, value, fee, 10)
+ parent_transaction = txid
+ for i in xrange(10):
+ transaction_package.append({'txid': txid, 'vout': i, 'amount': sent_value})
+
+ for i in xrange(1000):
+ utxo = transaction_package.pop(0)
+ try:
+ (txid, sent_value) = self.chain_transaction(utxo['txid'], utxo['vout'], utxo['amount'], fee, 10)
+ for j in xrange(10):
+ transaction_package.append({'txid': txid, 'vout': j, 'amount': sent_value})
+ if i == 998:
+ mempool = self.nodes[0].getrawmempool(True)
+ assert_equal(mempool[parent_transaction]['descendantcount'], 1000)
+ except JSONRPCException as e:
+ print e.error['message']
+ assert_equal(i, 999)
+ print "tx that would create too large descendant package successfully rejected"
+
+ # TODO: test descendant size limits
+
+if __name__ == '__main__':
+ MempoolPackagesTest().main()
View
@@ -411,6 +411,10 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-fuzzmessagestest=<n>", "Randomly fuzz 1 of every <n> network messages");
strUsage += HelpMessageOpt("-flushwallet", strprintf("Run a thread to flush wallet periodically (default: %u)", 1));
strUsage += HelpMessageOpt("-stopafterblockimport", strprintf("Stop running after importing blocks from disk (default: %u)", 0));
+ strUsage += HelpMessageOpt("-limitancestorcount=<n>", strprintf("Do not accept transactions if number of in-mempool ancestors is <n> or more (default: %u)", DEFAULT_ANCESTOR_LIMIT));
+ strUsage += HelpMessageOpt("-limitancestorsize=<n>", strprintf("Do not accept transactions whose size with all in-mempool ancestors exceeds <n> kilobytes (default: %u)", DEFAULT_ANCESTOR_SIZE_LIMIT));
+ strUsage += HelpMessageOpt("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT));
+ strUsage += HelpMessageOpt("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT));
}
string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, mempoolrej, net, proxy, prune, http"; // Don't translate these and qt below
if (mode == HMM_BITCOIN_QT)
View
@@ -921,6 +921,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
REJECT_HIGHFEE, "absurdly-high-fee",
strprintf("%d > %d", nFees, ::minRelayTxFee.GetFee(nSize) * 10000));
+ // Calculate in-mempool ancestors, up to a limit.
+ CTxMemPool::setEntries setAncestors;
+ size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT);
+ size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000;
+ size_t nLimitDescendants = GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT);
+ size_t nLimitDescendantSize = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000;
+ std::string errString;
+ if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) {
+ return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString);
+ }
+
// Check against previous transactions
// This is done last to help prevent CPU exhaustion denial-of-service attacks.
if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true))
@@ -942,7 +953,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
}
// Store transaction in memory
- pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
+ pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload());
}
SyncWithWallets(tx, NULL);
@@ -2033,13 +2044,23 @@ bool static DisconnectTip(CValidationState &state) {
if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED))
return false;
// Resurrect mempool transactions from the disconnected block.
+ std::vector<uint256> vHashUpdate;
BOOST_FOREACH(const CTransaction &tx, block.vtx) {
// ignore validation errors in resurrected transactions
list<CTransaction> removed;
CValidationState stateDummy;
- if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL))
+ if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) {
mempool.remove(tx, removed, true);
+ } else if (mempool.exists(tx.GetHash())) {
+ vHashUpdate.push_back(tx.GetHash());
+ }
}
+ // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have
+ // no in-mempool children, which is generally not true when adding
+ // previously-confirmed transactions back to the mempool.
+ // UpdateTransactionsFromBlock finds descendants of any transactions in this
+ // block that were added back and cleans up the mempool state.
+ mempool.UpdateTransactionsFromBlock(vHashUpdate);
mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight);
mempool.check(pcoinsTip);
// Update chainActive and related variables.
@@ -4258,7 +4279,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
LogPrint("mempool", "AcceptToMemoryPool: peer=%d %s: accepted %s (poolsz %u)\n",
pfrom->id, pfrom->cleanSubVer,
tx.GetHash().ToString(),
- mempool.mapTx.size());
+ mempool.size());
// Recursively process any orphan transactions that depended on this one
set<NodeId> setMisbehaving;
View
@@ -43,6 +43,14 @@ struct CNodeStateStats;
static const bool DEFAULT_ALERTS = true;
/** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
+/** Default for -limitancestorcount, max number of in-mempool ancestors */
+static const unsigned int DEFAULT_ANCESTOR_LIMIT = 100;
+/** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */
+static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 900;
+/** Default for -limitdescendantcount, max number of in-mempool descendants */
+static const unsigned int DEFAULT_DESCENDANT_LIMIT = 1000;
+/** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */
+static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 2500;
/** The maximum size of a blk?????.dat file (since 0.8) */
static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */
View
@@ -74,18 +74,30 @@ static inline size_t DynamicUsage(const std::vector<X>& v)
return MallocUsage(v.capacity() * sizeof(X));
}
-template<typename X>
-static inline size_t DynamicUsage(const std::set<X>& s)
+template<typename X, typename Y>
+static inline size_t DynamicUsage(const std::set<X, Y>& s)
{
return MallocUsage(sizeof(stl_tree_node<X>)) * s.size();
}
template<typename X, typename Y>
-static inline size_t DynamicUsage(const std::map<X, Y>& m)
+static inline size_t IncrementalDynamicUsage(const std::set<X, Y>& s)
+{
+ return MallocUsage(sizeof(stl_tree_node<X>));
+}
+
+template<typename X, typename Y, typename Z>
+static inline size_t DynamicUsage(const std::map<X, Y, Z>& m)
{
return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size();
}
+template<typename X, typename Y, typename Z>
+static inline size_t IncrementalDynamicUsage(const std::map<X, Y, Z>& m)
+{
+ return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >));
+}
+
// Boost data structures
template<typename X>
View
@@ -158,10 +158,10 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn)
// This vector will be sorted into a priority queue:
vector<TxPriority> vecPriority;
vecPriority.reserve(mempool.mapTx.size());
- for (map<uint256, CTxMemPoolEntry>::iterator mi = mempool.mapTx.begin();
+ for (CTxMemPool::indexed_transaction_set::iterator mi = mempool.mapTx.begin();
mi != mempool.mapTx.end(); ++mi)
{
- const CTransaction& tx = mi->second.GetTx();
+ const CTransaction& tx = mi->GetTx();
if (tx.IsCoinBase() || !IsFinalTx(tx, nHeight, pblock->nTime))
continue;
@@ -196,7 +196,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn)
}
mapDependers[txin.prevout.hash].push_back(porphan);
porphan->setDependsOn.insert(txin.prevout.hash);
- nTotalIn += mempool.mapTx[txin.prevout.hash].GetTx().vout[txin.prevout.n].nValue;
+ nTotalIn += mempool.mapTx.find(txin.prevout.hash)->GetTx().vout[txin.prevout.n].nValue;
continue;
}
const CCoins* coins = view.AccessCoins(txin.prevout.hash);
@@ -226,7 +226,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn)
porphan->feeRate = feeRate;
}
else
- vecPriority.push_back(TxPriority(dPriority, feeRate, &mi->second.GetTx()));
+ vecPriority.push_back(TxPriority(dPriority, feeRate, &(mi->GetTx())));
}
// Collect transactions into block
View
@@ -181,17 +181,19 @@ UniValue mempoolToJSON(bool fVerbose = false)
{
LOCK(mempool.cs);
UniValue o(UniValue::VOBJ);
- BOOST_FOREACH(const PAIRTYPE(uint256, CTxMemPoolEntry)& entry, mempool.mapTx)
+ BOOST_FOREACH(const CTxMemPoolEntry& e, mempool.mapTx)
{
- const uint256& hash = entry.first;
- const CTxMemPoolEntry& e = entry.second;
+ const uint256& hash = e.GetTx().GetHash();
UniValue info(UniValue::VOBJ);
info.push_back(Pair("size", (int)e.GetTxSize()));
info.push_back(Pair("fee", ValueFromAmount(e.GetFee())));
info.push_back(Pair("time", e.GetTime()));
info.push_back(Pair("height", (int)e.GetHeight()));
info.push_back(Pair("startingpriority", e.GetPriority(e.GetHeight())));
info.push_back(Pair("currentpriority", e.GetPriority(chainActive.Height())));
+ info.push_back(Pair("descendantcount", e.GetCountWithDescendants()));
+ info.push_back(Pair("descendantsize", e.GetSizeWithDescendants()));
+ info.push_back(Pair("descendantfees", e.GetFeesWithDescendants()));
const CTransaction& tx = e.GetTx();
set<string> setDepends;
BOOST_FOREACH(const CTxIn& txin, tx.vin)
@@ -246,6 +248,9 @@ UniValue getrawmempool(const UniValue& params, bool fHelp)
" \"height\" : n, (numeric) block height when transaction entered pool\n"
" \"startingpriority\" : n, (numeric) priority when transaction entered pool\n"
" \"currentpriority\" : n, (numeric) transaction priority now\n"
+ " \"descendantcount\" : n, (numeric) number of in-mempool descendant transactions (including this one)\n"
+ " \"descendantsize\" : n, (numeric) size of in-mempool descendants (including this one)\n"
+ " \"descendantfees\" : n, (numeric) fees of in-mempool descendants (including this one)\n"
" \"depends\" : [ (array) unconfirmed transactions used as inputs for this transaction\n"
" \"transactionid\", (string) parent transaction id\n"
" ... ]\n"
Oops, something went wrong.

0 comments on commit b0ce450

Please sign in to comment.