Floating fees for the client

Estimate what fee or priority is needed to get a transaction
accepted into a block (unless user has set a fee using
-paytxfee or the GUI).

Includes a safety net: always pay at least what a 0.8.5 client
would pay. The plan is for future releases to eliminate that
check, and let fees float up and down based on competition for
block space.

Adds two new RPC methods to expose the estimates:
estimatefee and estimatepriority.

I expect future releases might improve the fee estimation code;
it is self-contained in the CMinerPolicyEstimator class, so
"patches welcome."

This is a much more conservative of the pull request I
submitted a couple months ago.

qa/rpc-tests/README.md

@@ -3,4 +3,6 @@ Regression tests of RPC interface
 
 wallet.sh : Test wallet send/receive code (see comments for details)
 
+estimatefee.sh : Test estimatefee/estimatepriority RPC calls
+
 util.sh : useful re-usable functions 

qa/rpc-tests/estimatefee.sh

@@ -0,0 +1,149 @@
+#!/usr/bin/env bash
+
+# Test estimatefee
+
+#set -o xtrace  # Uncomment to debug
+
+if [ $# -lt 1 ]; then
+        echo "Usage: $0 path_to_binaries"
+        echo "e.g. $0 ../../src"
+        exit 1
+fi
+
+echo "Tests can take more than 10 minutes to run; be patient."
+
+BITCOIND=${1}/bitcoind
+CLI=${1}/bitcoin-cli
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/util.sh"
+
+D=$(mktemp -d test.XXXXX)
+
+# Three nodes:
+#  B1 relays between the other two, runs with defaults,
+D1=${D}/node1
+CreateDataDir $D1 port=11000 rpcport=11001 debug=estimatefee
+B1ARGS="-datadir=$D1"
+$BITCOIND $B1ARGS &
+B1PID=$!
+
+#  B2 runs with -paytxfee=0.001, is also used to send
+#  higher/lower priority transactions
+D2=${D}/node2
+CreateDataDir $D2 port=11010 rpcport=11011 connect=127.0.0.1:11000 paytxfee=0.001 debug=estimatefee
+B2ARGS="-datadir=$D2"
+$BITCOIND $B2ARGS &
+B2PID=$!
+
+#  B3 runs with -paytxfee=0.02
+D3=${D}/node3
+CreateDataDir $D3 port=11020 rpcport=11021 connect=127.0.0.1:11000 paytxfee=0.02 debug=estimatefee
+B3ARGS="-datadir=$D3"
+$BITCOIND $BITCOINDARGS $B3ARGS &
+B3PID=$!
+
+# trap "kill -9 $B1PID $B2PID $B3PID; rm -rf $D" EXIT
+
+echo "Generating initial blockchain"
+
+# B2 starts with 220 blocks (so 120 old, mature):
+$CLI $B2ARGS setgenerate true 220
+WaitForBlock $B1ARGS 220
+
+PRI=$($CLI $B1ARGS estimatepriority)
+AssertEqual $PRI -1.0
+
+B1ADDRESS=$(Address $B1ARGS)
+B2ADDRESS=$(Address $B2ARGS)
+B3ADDRESS=$(Address $B3ARGS)
+
+echo "Testing priority estimation"
+
+# Generate 110 very-high-priority (50 BTC with at least 100 confirmations) transactions
+# (in two batches, with a block generated in between)
+# 100 confirmed transactions is the minimum the estimation code needs.
+for i in {1..55} ; do
+    RAW=$(CreateTxn1 $B2ARGS 1 $B1ADDRESS)
+    RAWTXID=$(SendRawTxn $B2ARGS $RAW)
+done
+$CLI $B3ARGS setgenerate true 1
+WaitForBlock $B1ARGS 221
+for i in {1..55} ; do
+    RAW=$(CreateTxn1 $B2ARGS 1 $B1ADDRESS)
+    RAWTXID=$(SendRawTxn $B2ARGS $RAW)
+done
+sleep 1
+$CLI $B3ARGS setgenerate true 3
+WaitForBlock $B1ARGS 224
+
+PRI_HIGH=$($CLI $B3ARGS estimatepriority)
+AssertGreater "$PRI_HIGH" 56000000
+
+# Generate 100 very-low-priority free transactions;
+# priority estimate should go down
+
+for i in {1..50} ; do
+    RAW=$(CreateTxn1 $B1ARGS 1 $B2ADDRESS)
+    RAWTXID=$(SendRawTxn $B1ARGS $RAW)
+    RAW=$(CreateTxn1 $B1ARGS 1 $B3ADDRESS)
+    RAWTXID=$(SendRawTxn $B1ARGS $RAW)
+done
+$CLI $B3ARGS setgenerate true 3
+WaitForBlock $B1ARGS 227
+
+PRI_LOW=$($CLI $B3ARGS estimatepriority)
+AssertGreater "$PRI_HIGH" "$PRI_LOW"
+
+echo "Success. Testing fee estimation."
+
+# At this point:
+#  B1 has 60 unspent outputs with 6 or 7 confirmations
+#  B2 has 17 coinbases with over 100 confirmations
+#   and 50 utxos with 3 confirmations
+#  B3 has no mature coins/transactions.
+#
+# So: create 200 1 XBT transactions from B2 to B3,
+# using sendtoaddress.  The first 67 will be free, the
+# rest will pay a 0.001 XBT fee, and estimatefee
+# should give an estimate of over 0.00099 XBT/kilobyte
+#  (transactions are less than 1KB big)
+for i in {1..200} ; do
+    Send $B2ARGS $B3ARGS 1
+done
+$CLI $B2ARGS setgenerate true 1
+WaitForBlock $B3ARGS 228
+
+FEE_LOW=$($CLI $B3ARGS estimatefee)
+AssertGreater "$FEE_LOW" "0.00099"
+echo "Fee low: $FEE_LOW"
+
+# Now create 500 0.1 XBT transactions
+# from B3 to B1, each paying a 0.02 XBT fee.
+# At most 200 will have non-zero priority,
+# so at least 300 will be zero-priority.
+# Median fee estimate should go up.
+for i in {1..500} ; do
+    Send $B3ARGS $B1ARGS 0.1
+done
+$CLI $B2ARGS setgenerate true 2
+WaitForBlock $B1ARGS 230
+
+FEE_HIGH=$($CLI $B2ARGS estimatefee)
+echo "Fee high: $FEE_HIGH"
+AssertGreater "$FEE_HIGH" "$FEE_LOW"
+
+# Shut down, clean up
+
+$CLI $B3ARGS stop > /dev/null 2>&1
+wait $B3PID
+$CLI $B2ARGS stop > /dev/null 2>&1
+wait $B2PID
+$CLI $B1ARGS stop > /dev/null 2>&1
+wait $B1PID
+
+echo "Tests successful, cleaning up"
+trap "" EXIT
+rm -rf $D
+exit 0

qa/rpc-tests/util.sh

@@ -30,6 +30,14 @@ function CreateDataDir {
   done
 }
 
+function AssertNotEmpty {
+  if [ -z  "$1" ]
+  then
+    echoerr "Unexpected empty result."
+    exit 1
+  fi
+}
+
 function AssertEqual {
   if (( $( echo "$1 == $2" | bc ) == 0 ))
   then
@@ -38,6 +46,22 @@ function AssertEqual {
   fi
 }
 
+function AssertGreater {
+  if (( $( echo "$1 > $2" | bc ) == 0 ))
+  then
+    echoerr "AssertGreater: $1 <= $2"
+    exit 1
+  fi
+}
+
+# WaitBlock -datadir=... block#
+function WaitForBlock {
+  while [ $( $CLI $1 getblockcount) -lt $2 ]
+  do
+    sleep 1
+  done
+}
+
 # CheckBalance -datadir=... amount account minconf
 function CheckBalance {
   B=$( $CLI $1 getbalance $3 $4 )
@@ -60,6 +84,7 @@ function Send {
   amount=$3
   address=$(Address $to)
   txid=$( $CLI $from sendtoaddress $address $amount )
+  AssertNotEmpty "$txid"
 }
 
 # Use: Unspent <datadir> <n'th-last-unspent> <var>
@@ -75,10 +100,13 @@ function CreateTxn1 {
   AMOUNT=$(Unspent $1 $2 amount)
   VOUT=$(Unspent $1 $2 vout)
   RAWTXN=$( $CLI $1 createrawtransaction "[{\"txid\":\"$TXID\",\"vout\":$VOUT}]" "{\"$3\":$AMOUNT}")
+  AssertNotEmpty "$RAWTXN"
   ExtractKey hex "$( $CLI $1 signrawtransaction $RAWTXN )"
 }
 
 # Use: SendRawTxn <datadir> <hex_txn_data>
 function SendRawTxn {
-  $CLI $1 sendrawtransaction $2
+  TXID=$($CLI $1 sendrawtransaction $2)
+  AssertNotEmpty "$TXID"
+  echo $TXID
 }

src/main.cpp

@@ -587,36 +587,86 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state)
     return true;
 }
 
-int64_t GetMinFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree, enum GetMinFee_mode mode)
+bool AllowFree(const CTransaction& tx, double dPriority, unsigned int nBytes, enum GetMinFee_mode mode)
 {
-    // Base fee is either nMinTxFee or nMinRelayTxFee
-    int64_t nBaseFee = (mode == GMF_RELAY) ? tx.nMinRelayTxFee : tx.nMinTxFee;
+    // There is a free transaction area in blocks created by most miners,
+    // * If we are relaying we allow transactions up to DEFAULT_BLOCK_PRIORITY_SIZE - 1000
+    //   to be considered to fall into this category. We don't want to encourage sending
+    //   multiple transactions instead of one big transaction to avoid fees.
+    // * If we are creating a transaction we allow transactions up to 1,000 bytes
+    //   to be considered safe and assume they can likely make it into this section.
+    if (nBytes >= (mode == GMF_SEND ? 1000 : (DEFAULT_BLOCK_PRIORITY_SIZE - 1000)))
+        return false;
+
+    if (mode == GMF_RELAY)
+        return true; // Relay almost everything (free txn rate-limiter keeps spam under control)
+
+    // This check can be removed after enough miners have upgraded to version 0.9.
+    // Until then, be safe when sending and require a fee if any output
+    // is less than CENT:
+    BOOST_FOREACH(const CTxOut& txout, tx.vout)
+        if (txout.nValue < CENT) return false;
+
+    return dPriority >= mempool.estimateFreePriority(0.5, true);
+}
+
+// Minimum fee as of version 0.8.5.
+// Remove this when network has upgraded.
+static int64_t GetMinFee08(unsigned int nBytes, enum GetMinFee_mode mode)
+{
+    int64_t nBaseFee = (mode == GMF_RELAY) ? CTransaction::nMinRelayTxFee : CTransaction::nMinTxFee;
 
+    // nBytes is rounded up to the next nearest 1,000 bytes, and nBaseFee per
+    // 1,000 bytes is required.
     int64_t nMinFee = (1 + (int64_t)nBytes / 1000) * nBaseFee;
 
-    if (fAllowFree)
+    return nMinFee;
+}
+
+int64_t GetMinFee(unsigned int nBytes, enum GetMinFee_mode mode)
+{
+    // The fee rules have evolved over time, and are more complicated
+    // than we'd like. Changing them requires first changing miners,
+    // and then changing wallet software.
+    //
+    // Summary of fee-related settings:
+    // -paytxfee / ::nTransactionFee : User setting, means
+    //   "pay at least this much per transaction."  Default 0.
+    //   If -paytxfee is not set, then current fees are
+    //   estimated.
+    //
+    // -mintxfee / CTransaction::nMinTxFee : Miner setting,
+    //   means "transactions that pay less than this
+    //   per kilobyte considered free when creating blocks."
+    //
+    // -minrelaytxfee / CTransaction::nMinRelayTxFee : Relay
+    //   setting, means "transactions that pay less than
+    //   this per kb considered free when relaying."
+    //
+
+    int64_t nFee08 = GetMinFee08(nBytes, mode); // Remove when network has upgraded
+
+
+    int64_t nFee = 0;
+
+    if (mode == GMF_SEND)
     {
-        // There is a free transaction area in blocks created by most miners,
-        // * If we are relaying we allow transactions up to DEFAULT_BLOCK_PRIORITY_SIZE - 1000
-        //   to be considered to fall into this category. We don't want to encourage sending
-        //   multiple transactions instead of one big transaction to avoid fees.
-        // * If we are creating a transaction we allow transactions up to 1,000 bytes
-        //   to be considered safe and assume they can likely make it into this section.
-        if (nBytes < (mode == GMF_SEND ? 1000 : (DEFAULT_BLOCK_PRIORITY_SIZE - 1000)))
-            nMinFee = 0;
+        // If user set nTransactionFee, pay at least that
+        // (pay nTransactionFee per kb, if transaction is > 1,000 bytes):
+        if (nTransactionFee > 0)
+            nFee = max(nTransactionFee, nTransactionFee*nBytes/1000);
+        // Otherwise pay median estimated fee-per-byte:
+        else
+            nFee = nBytes*mempool.estimateFee(0.5, true);
     }
-
-    // This code can be removed after enough miners have upgraded to version 0.9.
-    // Until then, be safe when sending and require a fee if any output
-    // is less than CENT:
-    if (nMinFee < nBaseFee && mode == GMF_SEND)
+    else // Relay:
     {
-        BOOST_FOREACH(const CTxOut& txout, tx.vout)
-            if (txout.nValue < CENT)
-                nMinFee = nBaseFee;
+        nFee = nBytes*CTransaction::nMinRelayTxFee/1000;
     }
 
-    if (!MoneyRange(nMinFee))
+    int64_t nMinFee = max(nFee08, nFee);
+
+    if (!MoneyRange(nMinFee)) // belt-and-suspenders sanity check
         nMinFee = MAX_MONEY;
     return nMinFee;
 }
@@ -715,7 +765,9 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa
         unsigned int nSize = entry.GetTxSize();
 
         // Don't accept it if it can't get into a block
-        int64_t txMinFee = GetMinFee(tx, nSize, true, GMF_RELAY);
+        int64_t txMinFee = 0;
+        if (!AllowFree(tx, dPriority, nSize, GMF_RELAY))
+            txMinFee = GetMinFee(nSize, GMF_RELAY);
         if (fLimitFree && nFees < txMinFee)
             return state.DoS(0, error("AcceptToMemoryPool : not enough fees %s, %"PRId64" < %"PRId64,
                                       hash.ToString().c_str(), nFees, txMinFee),

src/main.h

@@ -255,7 +255,8 @@ enum GetMinFee_mode
     GMF_SEND,
 };
 
-int64_t GetMinFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree, enum GetMinFee_mode mode);
+bool AllowFree(const CTransaction& tx, double dPriority, unsigned int nBytes, enum GetMinFee_mode mode);
+int64_t GetMinFee(unsigned int nBytes, enum GetMinFee_mode mode);
 
 //
 // Check transaction inputs, and make sure any
@@ -290,13 +291,6 @@ unsigned int GetLegacySigOpCount(const CTransaction& tx);
 unsigned int GetP2SHSigOpCount(const CTransaction& tx, CCoinsViewCache& mapInputs);
 
 
-inline bool AllowFree(double dPriority)
-{
-    // Large (in bytes) low-priority (new, small-coin) transactions
-    // need a fee.
-    return dPriority > COIN * 144 / 250;
-}
-
 // Check whether all inputs of this transaction are valid (no double spends, scripts & sigs, amounts)
 // This does not modify the UTXO set. If pvChecks is not NULL, script checks are pushed onto it
 // instead of being performed inline.

src/miner.cpp

@@ -247,9 +247,6 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn)
             unsigned int nTxSize = ::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION);
             dPriority = tx.ComputePriority(dPriority, nTxSize);
 
-            // This is a more accurate fee-per-kilobyte than is used by the client code, because the
-            // client code rounds up the size to the nearest 1K. That's good, because it gives an
-            // incentive to create smaller transactions.
             double dFeePerKb =  double(nTotalIn-tx.GetValueOut()) / (double(nTxSize)/1000.0);
 
             if (porphan)
@@ -297,7 +294,7 @@ CBlockTemplate* CreateNewBlock(const CScript& scriptPubKeyIn)
             // Prioritize by fee once past the priority size or we run out of high-priority
             // transactions:
             if (!fSortedByFee &&
-                ((nBlockSize + nTxSize >= nBlockPrioritySize) || !AllowFree(dPriority)))
+                ((nBlockSize + nTxSize >= nBlockPrioritySize) || !AllowFree(tx, dPriority, nTxSize, GMF_SEND)))
             {
                 fSortedByFee = true;
                 comparer = TxPriorityCompare(fSortedByFee);