Save/restore memory pool

src/init.cpp

@@ -120,6 +120,7 @@ void Shutdown()
     GenerateBitcoins(false, NULL, 0);
 #endif
     StopNode();
+    mempool.Write();
     {
         LOCK(cs_main);
 #ifdef ENABLE_WALLET
@@ -895,6 +896,21 @@ bool AppInit2(boost::thread_group& threadGroup, bool fForceServer)
         return false;
     }
 
+    // It is OK if mempool.Read() fails; starting out with an empty memory pool is not
+    // a problem, it gets filled quickly.
+    list<CTxMemPoolEntry> mempoolEntries;
+    if (mempool.Read(mempoolEntries) && !empty(mempoolEntries))
+    {
+        CValidationState valState;
+        bool fMissingInputs;
+        BOOST_FOREACH(CTxMemPoolEntry& mempoolEntry, mempoolEntries)
+        {
+            AcceptToMemoryPool(mempool, valState, mempoolEntry.GetTx(), false,
+                               &fMissingInputs, false);
+        }
+        LogPrintf("Accepted %lu mempool transactions\n", mempool.size());
+    }
+
     // ********************************************************* Step 8: load wallet
 #ifdef ENABLE_WALLET
     if (fDisableWallet) {

src/main.cpp

@@ -1731,7 +1731,6 @@ bool SetBestChain(CValidationState &state, CBlockIndex* pindexNew)
     }
 
     // Connect longer branch
-    vector<CTransaction> vDelete;
     BOOST_FOREACH(CBlockIndex *pindex, vConnect) {
         CBlock block;
         if (!ReadBlockFromDisk(block, pindex))
@@ -1747,9 +1746,12 @@ bool SetBestChain(CValidationState &state, CBlockIndex* pindexNew)
         if (fBenchmark)
             LogPrintf("- Connect: %.2fms\n", (GetTimeMicros() - nStart) * 0.001);
 
-        // Queue memory transactions to delete
+        // Accepted into block, means remove from memory pool
         BOOST_FOREACH(const CTransaction& tx, block.vtx)
-            vDelete.push_back(tx);
+        {
+            mempool.remove(tx, false, pindex->nHeight-1);
+            mempool.removeConflicts(tx);
+        }
     }
 
     // Flush changes to global coin state
@@ -1792,12 +1794,6 @@ bool SetBestChain(CValidationState &state, CBlockIndex* pindexNew)
             mempool.remove(tx, true);
     }
 
-    // Delete redundant memory transactions that are in the connected branch
-    BOOST_FOREACH(CTransaction& tx, vDelete) {
-        mempool.remove(tx);
-        mempool.removeConflicts(tx);
-    }
-
     mempool.check(pcoinsTip);
 
     // Update best block in wallet (so we can detect restored wallets)

src/txmempool.cpp

@@ -3,10 +3,156 @@
 // Distributed under the MIT/X11 software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include "compat.h"
 #include "core.h"
 #include "txmempool.h"
 
+// These must come after compat.h to avoid windows
+// cross-compiler build errors.
+#include <boost/multi_index_container.hpp>
+#include <boost/multi_index/ordered_index.hpp>
+#include <boost/multi_index/member.hpp>
+
 using namespace std;
+using namespace boost;
+using namespace boost::multi_index;
+
+static const char* MEMPOOL_FILENAME="mempool.dat";
+
+// CMinerPolicyEstimator is told when transactions exit the
+// memory pool because they are included in blocks, and uses
+// that information to estimate the priority needed for
+// free transactions to be included in blocks and the
+// fee needed for fee-paying transactions.
+
+struct TimeValue
+{
+    int64_t t;
+    double v;
+    TimeValue(int64_t _t, double _v) : t(_t), v(_v) { }
+};
+typedef multi_index_container<
+    TimeValue,
+    indexed_by<
+        // Sort by time inserted
+        ordered_non_unique<member<TimeValue,int64_t,&TimeValue::t > >,
+
+        // Sort by value
+        ordered_non_unique<member<TimeValue,double,&TimeValue::v > >
+        >
+    > SortedValues ;
+
+class CMinerPolicyEstimator
+{
+private:
+    size_t nMin, nMax;
+    SortedValues byPriority;
+    SortedValues byFee;
+
+    // Results of estimate() are cached between new blocks, because
+    // the estimate doesn't change until a new block pulls transactions
+    // from the memory pool and because the transaction relaying code
+    // calls estimate on every transaction to decide whether or not it
+    // should be relayed.
+    map<double, double> byPriorityCache;
+    map<double, double> byFeeCache;
+
+    // Estimate what value is required to be chosen above
+    // fraction of other transactions (fraction is 0.0 to 1.0)
+    // Returns -1.0 if not enough data has been collected to
+    // give a good estimate.
+    double estimate(const SortedValues& values, double fraction, map<double,double>& cache)
+    {
+        assert(fraction >= 0.0 && fraction <= 1.0);
+        if (values.size() < nMin) return -1.0;
+
+        map<double,double>::iterator cached = cache.find(fraction);
+        if (cached != cache.end())
+            return cached->second;
+
+        size_t n = size_t(values.size()*fraction);
+        if (n > 0) --n;
+        SortedValues::nth_index<1>::type::iterator it=values.get<1>().begin();
+        std::advance(it, n);
+        cache[fraction] = it->v;
+        return it->v;
+    }
+
+    bool Write(CAutoFile& fileout, const SortedValues& values)
+    {
+        fileout << values.size();
+        SortedValues::nth_index<1>::type::iterator it=values.get<1>().begin();
+        while (it != values.get<1>().end())
+        {
+            fileout << it->t << it->v;
+            it++;
+        }
+        return true;
+    }
+    bool Read(CAutoFile& filein, SortedValues& values)
+    {
+        size_t n;
+        filein >> n;
+        for (size_t i = 0; i < n; i++)
+        {
+            int64_t t;
+            double v;
+            filein >> t >> v;
+            values.insert(TimeValue(t, v));
+        }
+        return true;
+    }
+
+public:
+    CMinerPolicyEstimator(size_t _nMin, size_t _nMax) : nMin(_nMin), nMax(_nMax)
+    {
+    }
+
+    void resize(SortedValues& what, size_t n)
+    {
+        while (what.size() > n)
+            what.erase(what.begin());
+    }
+
+    void add(const CTxMemPoolEntry& entry, unsigned int nBlockHeight)
+    {
+        if (nBlockHeight == 0 || entry.GetTxSize() == 0) return;
+        double dFeePerByte = entry.GetFee() / (double)entry.GetTxSize();
+        double dPriority = entry.GetPriority(nBlockHeight);
+        if (dPriority == 0 && dFeePerByte > 0)
+        {
+            byFee.insert(TimeValue(GetTimeMillis(), dFeePerByte));
+            resize(byFee, nMax);
+            byFeeCache.clear();
+        }
+        else if (dFeePerByte == 0 && dPriority > 0)
+        {
+            byPriority.insert(TimeValue(GetTimeMillis(), dPriority));
+            resize(byPriority, nMax);
+            byPriorityCache.clear();
+        }
+        // Ignore transactions with both fee and priority > 0,
+        // because we can't tell why miners included them (might
+        // have been priority, might have been fee)
+    }
+
+    double estimatePriority(double fraction)
+    {
+        return estimate(byPriority, fraction, byPriorityCache);
+    }
+    double estimateFee(double fraction)
+    {
+        return estimate(byFee, fraction, byFeeCache);
+    }
+    bool Write(CAutoFile& fileout)
+    {
+        return Write(fileout, byPriority) && Write(fileout, byFee);
+    }
+    bool Read(CAutoFile& filein)
+    {
+        return Read(filein, byPriority) && Read(filein, byFee);
+    }
+};
 
 CTxMemPoolEntry::CTxMemPoolEntry()
 {
@@ -41,6 +187,15 @@ CTxMemPool::CTxMemPool()
     // accepting transactions becomes O(N^2) where N is the number
     // of transactions in the pool
     fSanityCheck = false;
+    // 100 and 10,000 values here are arbitrary, but work
+    // well in practice, giving reasonable estimates within a few
+    // blocks and stable estimates over time.
+    minerPolicyEstimator = new CMinerPolicyEstimator(100, 10000);
+}
+
+CTxMemPool::~CTxMemPool()
+{
+    delete minerPolicyEstimator;
 }
 
 void CTxMemPool::pruneSpent(const uint256 &hashTx, CCoins &coins)
@@ -86,7 +241,7 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry)
 }
 
 
-bool CTxMemPool::remove(const CTransaction &tx, bool fRecursive)
+bool CTxMemPool::remove(const CTransaction &tx, bool fRecursive, unsigned int nBlockHeight)
 {
     // Remove transaction from memory pool
     {
@@ -101,6 +256,7 @@ bool CTxMemPool::remove(const CTransaction &tx, bool fRecursive)
         }
         if (mapTx.count(hash))
         {
+            minerPolicyEstimator->add(mapTx[hash], nBlockHeight);
             BOOST_FOREACH(const CTxIn& txin, tx.vin)
                 mapNextTx.erase(txin.prevout);
             mapTx.erase(hash);
@@ -192,6 +348,141 @@ bool CTxMemPool::lookup(uint256 hash, CTransaction& result) const
     return true;
 }
 
+void CTxMemPool::estimateFees(double dPriorityMedian, double& dPriority,
+                              double dFeeMedian, double& dFee, bool fUseHardCoded)
+{
+    LOCK(cs);
+    dPriority = minerPolicyEstimator->estimatePriority(dPriorityMedian);
+    // Hard-coded priority is 1-BTC, 144-confirmation old, 250-byte txn:
+    if (dPriority < 0 && fUseHardCoded) dPriority = COIN*144 / 250;
+    dFee = minerPolicyEstimator->estimateFee(dFeeMedian);
+    // Hard-coded fee is 10,000 satoshis per kilobyte (10 satoshis per byte):
+    if (dFee < 0 && fUseHardCoded) dFee = 10.0;
+}
+
+bool CTxMemPool::isDust(const CTxOut& txout)
+{
+    // "Dust" is defined as outputs so small
+    // (in value) that they would require that
+    // more than 1/3 of their value in fees to
+    // have a reasonable chance of being accepted into
+    // a block right now.
+    // Fees are per-byte, so:
+    int nSize = (int)::GetSerializeSize(txout, SER_DISK,0);
+    // ... and assume it will need a 148-byte CTxIn to spend:
+    nSize += 148;
+
+    // Use 0.01 (lowest 1%) as threshold to estimate fee-per-byte:
+    double dMinFee, dUnused;
+    estimateFees(0.01, dUnused, 0.01, dMinFee, true);
+
+    // Need to pay more than 1/3 of value?
+    bool fIsDust = dMinFee*nSize > txout.nValue/3;
+    return fIsDust;
+}
+
+void CTxMemPool::writeEntry(CAutoFile& file, const uint256& txid, std::set<uint256>& alreadyWritten) const
+{
+    if (alreadyWritten.count(txid)) return;
+    alreadyWritten.insert(txid);
+    const CTxMemPoolEntry& entry = mapTx.at(txid);
+    // Write txns we depend on first:
+    BOOST_FOREACH(const CTxIn txin, entry.GetTx().vin)
+    {
+        const uint256& prevout = txin.prevout.hash;
+        if (mapTx.count(prevout))
+            writeEntry(file, prevout, alreadyWritten);
+    }
+    unsigned int nHeight = entry.GetHeight();
+    file << entry.GetTx() << entry.GetFee() << entry.GetTime() << entry.GetPriority(nHeight) << nHeight;
+}
+
+//
+// Format of the mempool.dat file:
+//  32-bit versionRequiredToRead
+//  32-bit versionThatWrote
+//  32-bit-number of transactions
+//  [ serialized: transaction / fee / time / priority / height ]
+//    ... then the miner policy estimation data:
+//  32-bit-number of priority data points
+//  [ (time,priority) ]
+//  32-bit-number of fee data points
+//  [ (time,fee) ]
+//
+bool CTxMemPool::Write() const
+{
+    boost::filesystem::path path = GetDataDir() / MEMPOOL_FILENAME;
+    FILE *file = fopen(path.string().c_str(), "wb"); // Overwrites any older mempool (which is fine)
+    CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION);
+    if (!fileout)
+        return error("CTxMemPool::Write() : open failed");
+
+    fileout << CLIENT_VERSION; // version required to read
+    fileout << CLIENT_VERSION; // version that wrote the file
+
+    std::set<uint256> alreadyWritten; // Used to write parents before dependents
+    try {
+        LOCK(cs);
+        fileout << mapTx.size();
+        for (map<uint256, CTxMemPoolEntry>::const_iterator it = mapTx.begin();
+             it != mapTx.end(); it++)
+        {
+            writeEntry(fileout, it->first, alreadyWritten);
+        }
+        minerPolicyEstimator->Write(fileout);
+    }
+    catch (std::exception &e) {
+        // We don't care much about errors; saving
+        // and restoring the memory pool is mostly an
+        // optimization for cases where a mining node shuts down
+        // briefly (maybe to change an option), and it is better
+        // to restart with a full memory pool of transactions to mine.
+        return error("CTxMemPool::Write() : unable to write (non-fatal)");
+    }
+
+    return true;
+}
+
+bool CTxMemPool::Read(std::list<CTxMemPoolEntry>& vecEntries) const
+{
+    boost::filesystem::path path = GetDataDir() / MEMPOOL_FILENAME;
+    FILE *file = fopen(path.string().c_str(), "rb");
+    if (!file) return true; // No mempool.dat: OK
+    CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION);
+    if (!filein)
+        return error("CTxMemPool::Read() : open failed");
+
+    try {
+        int nVersionRequired, nVersionThatWrote;
+        filein >> nVersionRequired >> nVersionThatWrote;
+
+        if (nVersionRequired > CLIENT_VERSION)
+            return error("CTxMemPool::Read() : up-version (%d) mempool.dat", nVersionRequired);
+
+        size_t nTx;
+        filein >> nTx;
+
+        for (size_t i = 0; i < nTx; i++)
+        {
+            CTransaction tx;
+            int64_t nFee;
+            int64_t nTime;
+            double dPriority;
+            unsigned int nHeight;
+            filein >> tx >> nFee >> nTime >> dPriority >> nHeight;
+            CTxMemPoolEntry e(tx, 0, nTime, dPriority, nHeight);
+            vecEntries.push_back(e);
+        }
+        minerPolicyEstimator->Read(filein);
+    }
+    catch (std::exception &e) {
+        // Not a big deal if mempool.dat gets corrupted:
+        return error("CTxMemPool::Read() : unable to read (non-fatal)");
+    }
+
+    return true;
+}
+
 CCoinsViewMemPool::CCoinsViewMemPool(CCoinsView &baseIn, CTxMemPool &mempoolIn) : CCoinsViewBacked(baseIn), mempool(mempoolIn) { }
 
 bool CCoinsViewMemPool::GetCoins(const uint256 &txid, CCoins &coins) {