Implement on-the-fly mempool size limitation.

src/init.cpp

@@ -283,6 +283,7 @@ std::string HelpMessage(HelpMessageMode mode)
     strUsage += HelpMessageOpt("-dbcache=<n>", strprintf(_("Set database cache size in megabytes (%d to %d, default: %d)"), nMinDbCache, nMaxDbCache, nDefaultDbCache));
     strUsage += HelpMessageOpt("-loadblock=<file>", _("Imports blocks from external blk000??.dat file") + " " + _("on startup"));
     strUsage += HelpMessageOpt("-maxorphantx=<n>", strprintf(_("Keep at most <n> unconnectable transactions in memory (default: %u)"), DEFAULT_MAX_ORPHAN_TRANSACTIONS));
+    strUsage += HelpMessageOpt("-maxmempool=<n>", strprintf(_("Keep the transaction memory pool below <n> megabytes (default: %u)"), DEFAULT_MAX_MEMPOOL_SIZE));
     strUsage += HelpMessageOpt("-par=<n>", strprintf(_("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)"),
         -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS));
 #ifndef WIN32

src/main.cpp

@@ -859,22 +859,39 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
         CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), mempool.HasNoInputsOf(tx));
         unsigned int nSize = entry.GetTxSize();
 
+        // Try to make space in mempool
+        std::set<uint256> protect;
+        BOOST_FOREACH(const CTxIn& in, tx.vin) {
+            protect.insert(in.prevout.hash);
+        }
+        std::set<uint256> stagedelete;
+        CAmount nFeesDeleted = 0;
+        size_t nSizeDeleted = 0;
+        if (!mempool.StageTrimToSize(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, nFees, entry.GetFeeRate(), protect, stagedelete, nFeesDeleted, nSizeDeleted)) {
+            return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool-full");
+        }
+
+        if ((double)nFeesDeleted * nSize > (double)nFees * nSizeDeleted) {
+            // The new transaction's feerate is below that of the replaced transactions.
+            return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool-full-no-improve");
+        }
+
         // Don't accept it if it can't get into a block
         CAmount txMinFee = GetMinRelayFee(tx, nSize, true);
-        if (fLimitFree && nFees < txMinFee)
+        if (fLimitFree && nFees - nFeesDeleted < txMinFee)
             return state.DoS(0, error("AcceptToMemoryPool: not enough fees %s, %d < %d",
                                       hash.ToString(), nFees, txMinFee),
                              REJECT_INSUFFICIENTFEE, "insufficient fee");
 
         // Require that free transactions have sufficient priority to be mined in the next block.
-        if (GetBoolArg("-relaypriority", true) && nFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) {
+        if (GetBoolArg("-relaypriority", true) && nFees - nFeesDeleted < ::minRelayTxFee.GetFee(nSize) && !AllowFree(view.GetPriority(tx, chainActive.Height() + 1))) {
             return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority");
         }
 
         // Continuously rate-limit free (really, very-low-fee) transactions
         // This mitigates 'penny-flooding' -- sending thousands of free transactions just to
         // be annoying or make others' transactions take longer to confirm.
-        if (fLimitFree && nFees < ::minRelayTxFee.GetFee(nSize))
+        if (fLimitFree && nFees - nFeesDeleted < ::minRelayTxFee.GetFee(nSize))
         {
             static CCriticalSection csFreeLimiter;
             static double dFreeCount;
@@ -891,8 +908,8 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
             if (dFreeCount >= GetArg("-limitfreerelay", 15)*10*1000)
                 return state.DoS(0, error("AcceptToMemoryPool: free transaction rejected by rate limiter"),
                                  REJECT_INSUFFICIENTFEE, "rate limited free transaction");
-            LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount+nSize);
-            dFreeCount += nSize;
+            LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount+nSize+nSizeDeleted);
+            dFreeCount += nSize+nSizeDeleted;
         }
 
         if (fRejectAbsurdFee && nFees > ::minRelayTxFee.GetFee(nSize) * 10000)
@@ -921,6 +938,12 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
             return error("AcceptToMemoryPool: BUG! PLEASE REPORT THIS! ConnectInputs failed against MANDATORY but not STANDARD flags %s", hash.ToString());
         }
 
+        // Make actually space
+        if (!stagedelete.empty()) {
+            LogPrint("mempool", "Removing %u transactions (%d fees, %d bytes) from the mempool to make space for %s\n", stagedelete.size(), nFeesDeleted, nSizeDeleted, tx.GetHash().ToString());
+            pool.RemoveStaged(stagedelete);
+        }
+
         // Store transaction in memory
         pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
     }

src/main.h

@@ -50,6 +50,8 @@ struct CNodeStateStats;
 static const bool DEFAULT_ALERTS = true;
 /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */
 static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100;
+/** Default for -maxmempool, maximum megabytes of mempool memory usage */
+static const unsigned int DEFAULT_MAX_MEMPOOL_SIZE = 300;
 /** The maximum size of a blk?????.dat file (since 0.8) */
 static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB
 /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */

src/memusage.h

@@ -105,6 +105,12 @@ static inline size_t DynamicUsage(const std::set<X>& s)
     return MallocUsage(sizeof(stl_tree_node<X>)) * s.size();
 }
 
+template<typename X>
+static inline size_t IncrementalDynamicUsage(const std::set<X>& s)
+{
+    return MallocUsage(sizeof(stl_tree_node<X>));
+}
+
 template<typename X>
 static inline size_t RecursiveDynamicUsage(const std::set<X>& v)
 {
@@ -121,6 +127,12 @@ static inline size_t DynamicUsage(const std::map<X, Y>& m)
     return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >)) * m.size();
 }
 
+template<typename X, typename Y>
+static inline size_t IncrementalDynamicUsage(const std::map<X, Y>& m)
+{
+    return MallocUsage(sizeof(stl_tree_node<std::pair<const X, Y> >));
+}
+
 template<typename X, typename Y>
 static inline size_t RecursiveDynamicUsage(const std::map<X, Y>& v)
 {

src/txmempool.cpp

@@ -49,9 +49,10 @@ CTxMemPoolEntry::GetPriority(unsigned int currentHeight) const
     return dResult;
 }
 
-CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee) :
-    nTransactionsUpdated(0)
+CTxMemPool::CTxMemPool(const CFeeRate& _minRelayFee)
 {
+    clear();
+
     // Sanity checks off by default for performance, because otherwise
     // accepting transactions becomes O(N^2) where N is the number
     // of transactions in the pool
@@ -109,6 +110,19 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry,
     return true;
 }
 
+void CTxMemPool::removeUnchecked(const uint256& hash)
+{
+    indexed_transaction_set::iterator it = mapTx.find(hash);
+
+    BOOST_FOREACH(const CTxIn& txin, it->GetTx().vin)
+        mapNextTx.erase(txin.prevout);
+
+    totalTxSize -= it->GetTxSize();
+    cachedInnerUsage -= it->DynamicMemoryUsage();
+    mapTx.erase(it);
+    nTransactionsUpdated++;
+    minerPolicyEstimator->removeTx(hash);
+}
 
 void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& removed, bool fRecursive)
 {
@@ -144,15 +158,8 @@ void CTxMemPool::remove(const CTransaction &origTx, std::list<CTransaction>& rem
                     txToRemove.push_back(it->second.ptx->GetHash());
                 }
             }
-            BOOST_FOREACH(const CTxIn& txin, tx.vin)
-                mapNextTx.erase(txin.prevout);
-
             removed.push_back(tx);
-            totalTxSize -= mapTx.find(hash)->GetTxSize();
-            cachedInnerUsage -= mapTx.find(hash)->DynamicMemoryUsage();
-            mapTx.erase(hash);
-            nTransactionsUpdated++;
-            minerPolicyEstimator->removeTx(hash);
+            removeUnchecked(hash);
         }
     }
 }
@@ -435,3 +442,86 @@ size_t CTxMemPool::DynamicMemoryUsage() const {
     // Estimate the overhead of mapTx to be 5 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented.
     return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 5 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + cachedInnerUsage;
 }
+
+size_t CTxMemPool::GuessDynamicMemoryUsage(const CTxMemPoolEntry& entry) const {
+    return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 5 * sizeof(void*)) + entry.DynamicMemoryUsage() + memusage::IncrementalDynamicUsage(mapNextTx) * entry.GetTx().vin.size();
+}
+
+bool CTxMemPool::StageTrimToSize(size_t sizelimit, const CAmount& maxfeeremove, const CFeeRate& repfeerate, const std::set<uint256>& prot, std::set<uint256>& stage, CAmount& totalfeeremoved, size_t& totalsizeremoved) {
+    size_t expsize = DynamicMemoryUsage(); // Track the expected resulting memory usage of the mempool.
+    indexed_transaction_set::nth_index<1>::type::reverse_iterator it = mapTx.get<1>().rbegin();
+    int fails = 0; // Number of mempool transactions iterated over that were not included in the stage.
+    // Iterate from lowest feerate to highest feerate in the mempool:
+    while (expsize > sizelimit && it != mapTx.get<1>().rend()) {
+        const uint256& hash = it->GetTx().GetHash();
+        if (stage.count(hash)) {
+            // If the transaction is already staged for deletion, we know its descendants are already processed, so skip it.
+            it++;
+            continue;
+        }
+        if (it->GetFeeRate() > repfeerate) {
+            // If the transaction's feerate is worse than what we're looking for, we have processed everything in the mempool
+            // that could improve the staged set. If we don't have an acceptable solution by now, bail out.
+            return false;
+        }
+        std::deque<uint256> todo; // List of hashes that we still need to process (descendants of 'hash').
+        std::set<uint256> now; // Set of hashes that will need to be added to stage if 'hash' is included.
+        CAmount nowfee = 0; // Sum of the fees in 'now'.
+        size_t nowsize = 0; // Sum of the tx sizes in 'now'.
+        size_t nowusage = 0; // Sum of the memory usages of transactions in 'now'.
+        int iternow = 0; // Transactions we've inspected so far while determining whether 'hash' is acceptable.
+        todo.push_back(it->GetTx().GetHash()); // Add 'hash' to the todo list, to initiate processing its children.
+        bool good = true; // Whether including 'hash' (and all its descendants) is a good idea.
+        // Iterate breadth-first over all descendants of transaction with hash 'hash'.
+        while (!todo.empty()) {
+            uint256 hashnow = todo.front();
+            if (prot.count(hashnow)) {
+                // If this transaction is in the protected set, we're done with 'hash'.
+                good = false;
+                break;
+            }
+            iternow++; // We only count transactions we actually had to go find in the mempool.
+            const CTxMemPoolEntry* origTx = &*mapTx.find(hashnow);
+            nowfee += origTx->GetFee();
+            if (totalfeeremoved + nowfee > maxfeeremove) {
+                // If this pushes up to the total fees deleted too high, we're done with 'hash'.
+                good = false;
+                break;
+            }
+            todo.pop_front();
+            // Add 'hashnow' to the 'now' set, and update its statistics.
+            now.insert(hashnow);
+            nowusage += GuessDynamicMemoryUsage(*origTx);
+            nowsize += origTx->GetTxSize();
+            // Find dependencies of 'hashnow' and them to todo.
+            std::map<COutPoint, CInPoint>::iterator iter = mapNextTx.lower_bound(COutPoint(hashnow, 0));
+            while (iter != mapNextTx.end() && iter->first.hash == hashnow) {
+                const uint256& nexthash = iter->second.ptx->GetHash();
+                if (!(stage.count(nexthash) || now.count(nexthash))) {
+                    todo.push_back(nexthash);
+                }
+                iter++;
+            }
+        }
+        if (good) {
+            stage.insert(now.begin(), now.end());
+            totalfeeremoved += nowfee;
+            totalsizeremoved += nowsize;
+            expsize -= nowusage;
+        } else {
+            fails += iternow;
+            if (fails == 20) {
+                // Bail out after traversing 20 transactions that are not acceptable.
+                return false;
+            }
+        }
+        it++;
+    }
+    return true;
+}
+
+void CTxMemPool::RemoveStaged(std::set<uint256>& stage) {
+    BOOST_FOREACH(const uint256& hash, stage) {
+        removeUnchecked(hash);
+    }
+}

src/txmempool.h

@@ -59,8 +59,8 @@ class CTxMemPoolEntry
 
     const CTransaction& GetTx() const { return this->tx; }
     double GetPriority(unsigned int currentHeight) const;
-    CAmount GetFee() const { return nFee; }
-    CFeeRate GetFeeRate() const { return feeRate; }
+    const CAmount& GetFee() const { return nFee; }
+    const CFeeRate& GetFeeRate() const { return feeRate; }
     size_t GetTxSize() const { return nTxSize; }
     int64_t GetTime() const { return nTime; }
     unsigned int GetHeight() const { return nHeight; }
@@ -157,6 +157,7 @@ class CTxMemPool
     void setSanityCheck(bool _fSanityCheck) { fSanityCheck = _fSanityCheck; }
 
     bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true);
+    void removeUnchecked(const uint256& hash);
     void remove(const CTransaction &tx, std::list<CTransaction>& removed, bool fRecursive = false);
     void removeCoinbaseSpends(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight);
     void removeConflicts(const CTransaction &tx, std::list<CTransaction>& removed);
@@ -178,6 +179,15 @@ class CTxMemPool
     void ApplyDeltas(const uint256 hash, double &dPriorityDelta, CAmount &nFeeDelta);
     void ClearPrioritisation(const uint256 hash);
 
+    /** Build a list of transaction (hashes) to remove such that:
+     *  - The list is consistent (if a parent is included, all its dependencies are included as well).
+     *  - Removing said list will reduce the DynamicMemoryUsage below sizelimit.
+     *  - At most maxfeeremove worth of fees will be removed.
+     *  - No transaction whose hash is in prot will be removed.
+     */
+    bool StageTrimToSize(size_t sizelimit, const CAmount& maxfeeremove, const CFeeRate& repfeerate, const std::set<uint256>& prot, std::set<uint256>& stage, CAmount& totalfeeremoved, size_t& totalsizeremoved);
+    void RemoveStaged(std::set<uint256>& stage);
+
     unsigned long size()
     {
         LOCK(cs);
@@ -209,6 +219,7 @@ class CTxMemPool
     bool ReadFeeEstimates(CAutoFile& filein);
 
     size_t DynamicMemoryUsage() const;
+    size_t GuessDynamicMemoryUsage(const CTxMemPoolEntry& entry) const;
 };
 
 /**