Skip to content

Commit

Permalink
Merge #8179: Evict orphans which are included or precluded by accepte…
Browse files Browse the repository at this point in the history 
…d blocks.

54326a6 Increase maximum orphan size to 100,000 bytes. (Gregory Maxwell)
8c99d1b Treat orphans as implicit inv for parents, discard when parents rejected. (Gregory Maxwell)
11cc143 Adds an expiration time for orphan tx. (Gregory Maxwell)
db0ffe8 This eliminates the primary leak that causes the orphan map to  always grow to its maximum size. (Gregory Maxwell)
1b0bcc5 Track orphan by prev COutPoint rather than prev hash (Pieter Wuille)
  • Loading branch information
@laanwj
laanwj committed Jun 20, 2016
2 parents f6598df + 54326a6 commit 94ab58b
Show file tree
Hide file tree
Showing 3 changed files with 108 additions and 32 deletions.
134 changes: 103 additions & 31 deletions src/main.cpp
View file
Expand Up @@ -88,12 +88,22 @@ CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE;
CTxMemPool mempool(::minRelayTxFee);
FeeFilterRounder filterRounder(::minRelayTxFee);

struct IteratorComparator
{
template<typename I>
bool operator()(const I& a, const I& b)
{
return &(*a) < &(*b);
}
};

struct COrphanTx {
CTransaction tx;
NodeId fromPeer;
int64_t nTimeExpire;
};
map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main);
map<uint256, set<uint256> > mapOrphanTransactionsByPrev GUARDED_BY(cs_main);
map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main);
void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main);

/**
Expand Down Expand Up @@ -623,40 +633,42 @@ bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(c
// large transaction with a missing parent then we assume
// it will rebroadcast it later, after the parent transaction(s)
// have been mined or received.
// 10,000 orphans, each of which is at most 5,000 bytes big is
// at most 500 megabytes of orphans:
// 100 orphans, each of which is at most 99,999 bytes big is
// at most 10 megabytes of orphans and somewhat more byprev index (in the worst case):
unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION);
if (sz > 5000)
if (sz >= MAX_STANDARD_TX_SIZE)
{
LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString());
return false;
}

mapOrphanTransactions[hash].tx = tx;
mapOrphanTransactions[hash].fromPeer = peer;
BOOST_FOREACH(const CTxIn& txin, tx.vin)
mapOrphanTransactionsByPrev[txin.prevout.hash].insert(hash);
auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME});
assert(ret.second);
BOOST_FOREACH(const CTxIn& txin, tx.vin) {
mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first);
}

LogPrint("mempool", "stored orphan tx %s (mapsz %u prevsz %u)\n", hash.ToString(),
LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(),
mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size());
return true;
}

void static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
{
map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash);
if (it == mapOrphanTransactions.end())
return;
return 0;
BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin)
{
map<uint256, set<uint256> >::iterator itPrev = mapOrphanTransactionsByPrev.find(txin.prevout.hash);
auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout);
if (itPrev == mapOrphanTransactionsByPrev.end())
continue;
itPrev->second.erase(hash);
itPrev->second.erase(it);
if (itPrev->second.empty())
mapOrphanTransactionsByPrev.erase(itPrev);
}
mapOrphanTransactions.erase(it);
return 1;
}

void EraseOrphansFor(NodeId peer)
Expand All @@ -668,8 +680,7 @@ void EraseOrphansFor(NodeId peer)
map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid
if (maybeErase->second.fromPeer == peer)
{
EraseOrphanTx(maybeErase->second.tx.GetHash());
++nErased;
nErased += EraseOrphanTx(maybeErase->second.tx.GetHash());
}
}
if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer);
Expand All @@ -679,6 +690,26 @@ void EraseOrphansFor(NodeId peer)
unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main)
{
unsigned int nEvicted = 0;
static int64_t nNextSweep;
int64_t nNow = GetTime();
if (nNextSweep <= nNow) {
// Sweep out expired orphan pool entries:
int nErased = 0;
int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL;
map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin();
while (iter != mapOrphanTransactions.end())
{
map<uint256, COrphanTx>::iterator maybeErase = iter++;
if (maybeErase->second.nTimeExpire <= nNow) {
nErased += EraseOrphanTx(maybeErase->second.tx.GetHash());
} else {
nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime);
}
}
// Sweep again 5 minutes after the next entry that expires in order to batch the linear scan.
nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL;
if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased);
}
while (mapOrphanTransactions.size() > nMaxOrphans)
{
// Evict a random orphan:
Expand Down Expand Up @@ -2335,6 +2366,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin

CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL);

std::vector<uint256> vOrphanErase;
std::vector<int> prevheights;
CAmount nFees = 0;
int nInputs = 0;
Expand Down Expand Up @@ -2367,6 +2399,17 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight;
}

// Which orphan pool entries must we evict?
for (size_t j = 0; j < tx.vin.size(); j++) {
auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout);
if (itByPrev == mapOrphanTransactionsByPrev.end()) continue;
for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) {
const CTransaction& orphanTx = (*mi)->second.tx;
const uint256& orphanHash = orphanTx.GetHash();
vOrphanErase.push_back(orphanHash);
}
}

if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) {
return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__),
REJECT_INVALID, "bad-txns-nonfinal");
Expand Down Expand Up @@ -2454,6 +2497,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase);
hashPrevBestCoinBase = block.vtx[0].GetHash();

// Erase orphan transactions include or precluded by this block
if (vOrphanErase.size()) {
int nErased = 0;
BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) {
nErased += EraseOrphanTx(orphanHash);
}
LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased);
}

int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5;
LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001);

Expand Down Expand Up @@ -5041,7 +5093,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
return true;
}

vector<uint256> vWorkQueue;
deque<COutPoint> vWorkQueue;
vector<uint256> vEraseQueue;
CTransaction tx;
vRecv >> tx;
Expand All @@ -5060,7 +5112,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) {
mempool.check(pcoinsTip);
RelayTransaction(tx);
vWorkQueue.push_back(inv.hash);
for (unsigned int i = 0; i < tx.vout.size(); i++) {
vWorkQueue.emplace_back(inv.hash, i);
}

pfrom->nLastTXTime = GetTime();

Expand All @@ -5071,18 +5125,18 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,

// Recursively process any orphan transactions that depended on this one
set<NodeId> setMisbehaving;
for (unsigned int i = 0; i < vWorkQueue.size(); i++)
{
map<uint256, set<uint256> >::iterator itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue[i]);
while (!vWorkQueue.empty()) {
auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front());
vWorkQueue.pop_front();
if (itByPrev == mapOrphanTransactionsByPrev.end())
continue;
for (set<uint256>::iterator mi = itByPrev->second.begin();
for (auto mi = itByPrev->second.begin();
mi != itByPrev->second.end();
++mi)
{
const uint256& orphanHash = *mi;
const CTransaction& orphanTx = mapOrphanTransactions[orphanHash].tx;
NodeId fromPeer = mapOrphanTransactions[orphanHash].fromPeer;
const CTransaction& orphanTx = (*mi)->second.tx;
const uint256& orphanHash = orphanTx.GetHash();
NodeId fromPeer = (*mi)->second.fromPeer;
bool fMissingInputs2 = false;
// Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan
// resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get
Expand All @@ -5095,7 +5149,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) {
LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString());
RelayTransaction(orphanTx);
vWorkQueue.push_back(orphanHash);
for (unsigned int i = 0; i < orphanTx.vout.size(); i++) {
vWorkQueue.emplace_back(orphanHash, i);
}
vEraseQueue.push_back(orphanHash);
}
else if (!fMissingInputs2)
Expand Down Expand Up @@ -5124,13 +5180,29 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv,
}
else if (fMissingInputs)
{
AddOrphanTx(tx, pfrom->GetId());
bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected
BOOST_FOREACH(const CTxIn& txin, tx.vin) {
if (recentRejects->contains(txin.prevout.hash)) {
fRejectedParents = true;
break;
}
}
if (!fRejectedParents) {
BOOST_FOREACH(const CTxIn& txin, tx.vin) {
CInv inv(MSG_TX, txin.prevout.hash);
pfrom->AddInventoryKnown(inv);
if (!AlreadyHave(inv)) pfrom->AskFor(inv);
}
AddOrphanTx(tx, pfrom->GetId());

// DoS prevention: do not allow mapOrphanTransactions to grow unbounded
unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS));
unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx);
if (nEvicted > 0)
LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted);
// DoS prevention: do not allow mapOrphanTransactions to grow unbounded
unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS));
unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx);
if (nEvicted > 0)
LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted);
} else {
LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString());
}
} else {
assert(recentRejects);
recentRejects->insert(tx.GetHash());
Expand Down
4 changes: 4 additions & 0 deletions src/main.h
View file
Expand Up @@ -56,6 +56,10 @@ static const CAmount HIGH_TX_FEE_PER_KB = 0.01 * COIN;
static const CAmount HIGH_MAX_TX_FEE = 100 * HIGH_TX_FEE_PER_KB;
/** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
/** Expiration time for orphan transactions in seconds */
static const int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60;
/** Minimum time between orphan transactions expire time checks in seconds */
static const int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60;
/** Default for -limitancestorcount, max number of in-mempool ancestors */
static const unsigned int DEFAULT_ANCESTOR_LIMIT = 25;
/** Default for -limitancestorsize, maximum kilobytes of tx + all in-mempool ancestors */
Expand Down
2 changes: 1 addition & 1 deletion src/test/DoS_tests.cpp
View file
Expand Up @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans)
tx.vout.resize(1);
tx.vout[0].nValue = 1*CENT;
tx.vout[0].scriptPubKey = GetScriptForDestination(key.GetPubKey().GetID());
tx.vin.resize(500);
tx.vin.resize(2777);
for (unsigned int j = 0; j < tx.vin.size(); j++)
{
tx.vin[j].prevout.n = j;
Expand Down

0 comments on commit 94ab58b

Please sign in to comment.