Merge bitcoin#12257: [wallet] Use destination groups instead of coins…

… in coin select

232f96f doc: Add release notes for -avoidpartialspends (Karl-Johan Alm)
e00b469 clean-up: Remove no longer used ivars from CInputCoin (Karl-Johan Alm)
43e04d1 wallet: Remove deprecated OutputEligibleForSpending (Karl-Johan Alm)
0128121 test: Add basic testing for wallet groups (Karl-Johan Alm)
59d6f7b wallet: Switch to using output groups instead of coins in coin selection (Karl-Johan Alm)
87ebce2 wallet: Add output grouping (Karl-Johan Alm)
bb629cb Add -avoidpartialspends and m_avoid_partial_spends (Karl-Johan Alm)
65b3eda wallet: Add input bytes to CInputCoin (Karl-Johan Alm)
a443d7a moveonly: CoinElegibilityFilter into coinselection.h (Karl-Johan Alm)
173e18a utils: Add insert() convenience templates (Karl-Johan Alm)

Pull request description:

  This PR adds an optional (off by default) `-avoidpartialspends` flag, which changes coin select to use output groups rather than outputs, where each output group corresponds to all outputs with the same destination.

  It is a privacy improvement, as each time you spend some output, any other output that is publicly associated with the destination (address) will also be spent at the same time, at the cost of fee increase for cases where coin select without group restriction would find a more optimal set of coins (see example below).

  For regular use without address reuse, this PR should have no effect on the user experience whatsoever; it only affects users who, for some reason, have multiple outputs with the same destination (i.e. address reuse).

  Nodes with this turned off will still try to avoid partial spending, if the fee of the resulting transaction is not greater than the fee of the original transaction.

  Example: a node has four outputs linked to two addresses `A` and `B`:

  * 1.0 btc to `A`
  * 0.5 btc to `A`
  * 1.0 btc to `B`
  * 0.5 btc to `B`

  The node sends 0.2 btc to `C`. Without `-avoidpartialspends`, the following coin selection will occur:
  * 0.5 btc to `A` or `B` is picked
  * 0.2 btc is output to `C`
  * 0.3 - fee is output to (unique change address)

  With `-avoidpartialspends`, the following will instead happen:
  * Both of (0.5, 1.0) btc to `A` or `B` is picked (one or the other pair)
  * 0.2 btc is output to `C`
  * 1.3 - fee is output to (unique change address)

  As noted, the pro here is that, assuming nobody sends to the address after you spend from it, you will only ever use one address once. The con is that the transaction becomes slightly larger in this case, because it is overpicking outputs to adhere to the no partial spending rule.

  This complements bitcoin#10386, in particular it addresses @luke-jr and @gmaxwell's concerns in bitcoin#10386 (comment) and bitcoin#10386 (comment).

  Together with `-avoidreuse`, this fully addresses the concerns in bitcoin#10065 I believe.

Tree-SHA512: 24687a4490ba59cf4198ed90052944ff4996653a4257833bb52ed24d058b3e924800c9b3790aeb6be6385b653b49e304453e5d7ff960e64c682fc23bfc447621

# Conflicts:
#	src/Makefile.am
#	src/bench/coin_selection.cpp
#	src/wallet/coincontrol.h
#	src/wallet/coinselection.cpp
#	src/wallet/coinselection.h
#	src/wallet/init.cpp
#	src/wallet/test/coinselector_tests.cpp
#	src/wallet/wallet.cpp
#	src/wallet/wallet.h
#	test/functional/test_runner.py

src/Makefile.am

@@ -402,6 +402,7 @@ libdash_wallet_a_SOURCES = \
   coinjoin/coinjoin-util.cpp \
   interfaces/wallet.cpp \
   keepass.cpp \
+  wallet/coincontrol.cpp \
   wallet/crypter.cpp \
   wallet/db.cpp \
   wallet/fees.cpp \

src/bench/coin_selection.cpp

@@ -8,7 +8,7 @@
 
 #include <set>
 
-static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<COutput>& vCoins)
+static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<OutputGroup>& groups)
 {
     int nInput = 0;
 
@@ -21,7 +21,7 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<CO
 
     int nAge = 6 * 24;
     COutput output(wtx, nInput, nAge, true /* spendable */, true /* solvable */, true /* safe */);
-    vCoins.push_back(output);
+    groups.emplace_back(output.GetInputCoin(), 0, false, 0, 0);
 }
 
 // Simple benchmark for wallet coin selection. Note that it maybe be necessary
@@ -34,22 +34,22 @@ static void addCoin(const CAmount& nValue, const CWallet& wallet, std::vector<CO
 static void CoinSelection(benchmark::State& state)
 {
     const CWallet wallet(WalletLocation(), WalletDatabase::CreateDummy());
-    std::vector<COutput> vCoins;
+    std::vector<OutputGroup> groups;
     LOCK(wallet.cs_wallet);
 
     while (state.KeepRunning()) {
         // Add coins.
         for (int i = 0; i < 1000; i++)
-            addCoin(1000 * COIN, wallet, vCoins);
-        addCoin(3 * COIN, wallet, vCoins);
+            addCoin(1000 * COIN, wallet, groups);
+        addCoin(3 * COIN, wallet, groups);
 
         std::set<CInputCoin> setCoinsRet;
         CAmount nValueRet;
         bool bnb_used;
         CoinEligibilityFilter filter_standard(1, 6, 0);
         CoinSelectionParams coin_selection_params(false, 34, 148, CFeeRate(0), 0);
-        bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used)
-                       || wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, vCoins, setCoinsRet, nValueRet, coin_selection_params, bnb_used);
+        bool success = wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used)
+                       || wallet.SelectCoinsMinConf(1003 * COIN, filter_standard, groups, setCoinsRet, nValueRet, coin_selection_params, bnb_used);
         assert(success);
         assert(nValueRet == 1003 * COIN);
         assert(setCoinsRet.size() == 2);
@@ -63,17 +63,21 @@ static void CoinSelection(benchmark::State& state)
 }
 
 typedef std::set<CInputCoin> CoinSet;
+static const CWallet testWallet("dummy", WalletDatabase::CreateDummy());
+std::vector<std::unique_ptr<CWalletTx>> wtxn;
 
 // Copied from src/wallet/test/coinselector_tests.cpp
-static void add_coin(const CAmount& nValue, int nInput, std::vector<CInputCoin>& set)
+static void add_coin(const CAmount& nValue, int nInput, std::vector<OutputGroup>& set)
 {
     CMutableTransaction tx;
     tx.vout.resize(nInput + 1);
     tx.vout[nInput].nValue = nValue;
-    set.emplace_back(MakeTransactionRef(tx), nInput);
+    std::unique_ptr<CWalletTx> wtx(new CWalletTx(&testWallet, MakeTransactionRef(std::move(tx))));
+    set.emplace_back(COutput(wtx.get(), nInput, 0, true, true, true).GetInputCoin(), 0, true, 0, 0);
+    wtxn.emplace_back(std::move(wtx));
 }
 // Copied from src/wallet/test/coinselector_tests.cpp
-static CAmount make_hard_case(int utxos, std::vector<CInputCoin>& utxo_pool)
+static CAmount make_hard_case(int utxos, std::vector<OutputGroup>& utxo_pool)
 {
     utxo_pool.clear();
     CAmount target = 0;
@@ -88,7 +92,7 @@ static CAmount make_hard_case(int utxos, std::vector<CInputCoin>& utxo_pool)
 static void BnBExhaustion(benchmark::State& state)
 {
     // Setup
-    std::vector<CInputCoin> utxo_pool;
+    std::vector<OutputGroup> utxo_pool;
     CoinSet selection;
     CAmount value_ret = 0;
     CAmount not_input_fees = 0;

src/util.h

@@ -369,4 +369,18 @@ std::string CopyrightHolders(const std::string& strPrefix, unsigned int nStartYe
  */
 int ScheduleBatchPriority(void);
 
+namespace util {
+
+//! Simplification of std insertion
+template <typename Tdst, typename Tsrc>
+inline void insert(Tdst& dst, const Tsrc& src) {
+    dst.insert(dst.begin(), src.begin(), src.end());
+}
+template <typename TsetT, typename Tsrc>
+inline void insert(std::set<TsetT>& dst, const Tsrc& src) {
+    dst.insert(src.begin(), src.end());
+}
+
+} // namespace util
+
 #endif // BITCOIN_UTIL_H

src/wallet/coincontrol.cpp

@@ -0,0 +1,23 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <wallet/coincontrol.h>
+
+#include <util.h>
+
+void CCoinControl::SetNull()
+{
+    destChange = CNoDestination();
+    m_change_type.reset();
+    fAllowOtherInputs = false;
+    fAllowWatchOnly = false;
+    m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS);
+    setSelected.clear();
+    m_feerate.reset();
+    fOverrideFeeRate = false;
+    m_confirm_target.reset();
+    m_signal_bip125_rbf.reset();
+    m_fee_mode = FeeEstimateMode::UNSET;
+}
+

src/wallet/coincontrol.h

@@ -45,6 +45,8 @@ class CCoinControl
     boost::optional<CFeeRate> m_discard_feerate;
     //! Override the default confirmation target if set
     boost::optional<unsigned int> m_confirm_target;
+    //! Avoid partial use of funds sent to a given address
+    bool m_avoid_partial_spends;
     //! Fee estimation mode to control arguments to estimateSmartFee
     FeeEstimateMode m_fee_mode;
     //! Controls which types of coins are allowed to be used (default: ALL_COINS)
@@ -57,16 +59,8 @@ class CCoinControl
 
     void SetNull(bool fResetCoinType = true)
     {
-        destChange = CNoDestination();
-        fAllowOtherInputs = false;
         fRequireAllInputs = true;
-        fAllowWatchOnly = false;
-        setSelected.clear();
-        m_feerate.reset();
         m_discard_feerate.reset();
-        fOverrideFeeRate = false;
-        m_confirm_target.reset();
-        m_fee_mode = FeeEstimateMode::UNSET;
         if (fResetCoinType) {
             nCoinType = CoinType::ALL_COINS;
         }