Merge #1641: [Core] New fee estimation code

4038de4 [QA] Restore 'feature_fee_estimation.py' functional test (random-zebra)
988f349 [qt] Properly display required fee instead of minTxFee (random-zebra)
c5c9df0 [Trivial] Fix typos (random-zebra)
a8fa99b [Trivial] Remove redundat check in ContainsZerocoins (random-zebra)
8ebcbcd PolicyEstimator: exclude zerocoin spends when computing estimates (random-zebra)
b67049b Create new BlockPolicyEstimator for fee estimates (random-zebra)

Pull request description:

  This introduces the new fee and priority estimation code described here <https://www.mail-archive.com/bitcoin-development@lists.sourceforge.net/msg06405.html>.

  Instead of calculating the median fee for each possible number of blocks needed to confirm, the new code divides the possible fee rates into buckets (spaced logarithmically) and keeps track of the number of blocks needed to confirm for each transaction in each bucket.

  Backports:
  - bitcoin#5159
  - bitcoin#6887

  except for the functional test (`smartfees.py`) which is based on an older version of the framework.
  Instead I've restored a more recent `feature_fee_estimation.py` (commenting out the check for `estimatesmartfee` which can be reintroduced once bitcoin#6134 is backported).

  Additional commits exclude zerocoins txes from the estimates calculation, as they have fixed fee/priority.

ACKs for top commit:
  Fuzzbawls:
    ACK 4038de4
  furszy:
    ACK 4038de4 and merging

Tree-SHA512: 25777af469f7fa84d2dab991544392bea8418bccb4d2c23113de2c6ed7047891bdaedad1728cb04ba343e82f4bc0c1446f88e28def698dd25168769abf8620bb

CMakeLists.txt

@@ -301,6 +301,7 @@ set(SERVER_SOURCES
         ./src/miner.cpp
         ./src/net.cpp
         ./src/noui.cpp
+        ./src/policy/fees.cpp
         ./src/pow.cpp
         ./src/rest.cpp
         ./src/rpc/blockchain.cpp

src/Makefile.am

@@ -146,6 +146,7 @@ BITCOIN_CORE_H = \
   netbase.h \
   net.h \
   noui.h \
+  policy/fees.h \
   pow.h \
   prevector.h \
   protocol.h \
@@ -237,6 +238,7 @@ libbitcoin_server_a_SOURCES = \
   miner.cpp \
   net.cpp \
   noui.cpp \
+  policy/fees.cpp \
   pow.cpp \
   rest.cpp \
   rpc/blockchain.cpp \

src/Makefile.test.include

@@ -62,6 +62,7 @@ BITCOIN_TESTS =\
   test/net_tests.cpp \
   test/netbase_tests.cpp \
   test/pmt_tests.cpp \
+  test/policyestimator_tests.cpp \
   test/prevector_tests.cpp \
   test/random_tests.cpp \
   test/reverselock_tests.cpp \

src/init.cpp

@@ -516,7 +516,7 @@ std::string HelpMessage(HelpMessageMode mode)
         strUsage += HelpMessageOpt("-maxsigcachesize=<n>", strprintf(_("Limit size of signature cache to <n> entries (default: %u)"), 50000));
     }
     strUsage += HelpMessageOpt("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE));
-    strUsage += HelpMessageOpt("-minrelaytxfee=<amt>", strprintf(_("Fees (in %s/Kb) smaller than this are considered zero fee for relaying (default: %s)"), CURRENCY_UNIT, FormatMoney(::minRelayTxFee.GetFeePerK())));
+    strUsage += HelpMessageOpt("-minrelaytxfee=<amt>", strprintf(_("Fees (in %s/Kb) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)"), CURRENCY_UNIT, FormatMoney(::minRelayTxFee.GetFeePerK())));
     strUsage += HelpMessageOpt("-printtoconsole", strprintf(_("Send trace/debug info to console instead of debug.log file (default: %u)"), 0));
     if (GetBoolArg("-help-debug", false)) {
         strUsage += HelpMessageOpt("-printpriority", strprintf(_("Log transaction priority and fee per kB when mining blocks (default: %u)"), 0));

src/main.cpp

@@ -98,7 +98,7 @@ size_t nCoinCacheUsage = 5000 * 300;
 /* If the tip is older than this (in seconds), the node is considered to be in initial block download. */
 int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE;
 
-/** Fees smaller than this (in upiv) are considered zero fee (for relaying and mining)
+/** Fees smaller than this (in upiv) are considered zero fee (for relaying, mining and transaction creation)
  * We are ~100 times smaller then bitcoin now (2015-06-23), set minRelayTxFee only 10 times higher
  * so it's still 10 times lower comparing to bitcoin.
  */
@@ -1132,7 +1132,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState& state, const CTransa
         if (!hasZcSpendInputs)
             view.GetPriority(tx, chainHeight);
 
-        CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainHeight);
+        CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainHeight, mempool.HasNoInputsOf(tx));
         unsigned int nSize = entry.GetTxSize();
 
         // Don't accept it if it can't get into a block
@@ -1212,7 +1212,7 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState& state, const CTransa
         }
 
         // Store transaction in memory
-        pool.addUnchecked(hash, entry);
+        pool.addUnchecked(hash, entry, !IsInitialBlockDownload());
     }
 
     SyncWithWallets(tx, nullptr);
@@ -1329,7 +1329,7 @@ bool AcceptableInputs(CTxMemPool& pool, CValidationState& state, const CTransact
         CAmount nFees = nValueIn - nValueOut;
         double dPriority = view.GetPriority(tx, chainHeight);
 
-        CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainHeight);
+        CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainHeight, mempool.HasNoInputsOf(tx));
         unsigned int nSize = entry.GetTxSize();
 
         // Don't accept it if it can't get into a block
@@ -2825,7 +2825,7 @@ bool static ConnectTip(CValidationState& state, CBlockIndex* pindexNew, CBlock*
 
     // Remove conflicting transactions from the mempool.
     std::list<CTransaction> txConflicted;
-    mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted);
+    mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload());
     mempool.check(pcoinsTip);
     // Update chainActive & related variables.
     UpdateTip(pindexNew);