Add consolidate endpoint and adjust subtractfee logic (#235)

* Add consolidate endpoint and adjust subtractfee logic

* Fix and augment tests

* Update src/Stratis.Bitcoin.Features.Wallet/Services/WalletService.cs

src/NBitcoin/TransactionBuilder.cs

@@ -1545,7 +1545,7 @@ public ICoin[] FindSpentCoins(Transaction tx)
         /// Estimate the physical size of the transaction
         /// </summary>
         /// <param name="tx">The transaction to be estimated</param>
-        /// <returns></returns>
+        /// <returns>The estimated size of the transaction in bytes.</returns>
         public int EstimateSize(Transaction tx)
         {
             return EstimateSize(tx, false);
@@ -1617,7 +1617,7 @@ private void EstimateScriptSigSize(ICoin coin, ref int witSize, ref int baseSize
             }
 
             if (scriptSigSize == -1)
-                scriptSigSize += coin.TxOut.ScriptPubKey.Length; //Using heurestic to approximate size of unknown scriptPubKey
+                scriptSigSize += coin.TxOut.ScriptPubKey.Length; //Using heuristic to approximate size of unknown scriptPubKey
 
             if (coin.GetHashVersion(this.Network) == HashVersion.Witness)
                 witSize += scriptSigSize + 1; //Account for the push

src/Stratis.Bitcoin.Features.SmartContracts/Wallet/SmartContractWalletTransactionHandler.cs

@@ -57,8 +57,9 @@ protected override void AddRecipients(TransactionBuildContext context)
             if (context.Recipients.Any(recipient => recipient.Amount == Money.Zero && !recipient.ScriptPubKey.IsSmartContractExec()))
                 throw new WalletException("No amount specified.");
 
+            // TODO: Port the necessary logic from the regular wallet transaction handler
             if (context.Recipients.Any(a => a.SubtractFeeFromAmount))
-                throw new NotImplementedException("Substracting the fee from the recipient is not supported yet.");
+                throw new NotImplementedException("Subtracting the fee from the recipient is not supported yet.");
 
             foreach (Recipient recipient in context.Recipients)
                 context.TransactionBuilder.Send(recipient.ScriptPubKey, recipient.Amount);

src/Stratis.Bitcoin.Features.Wallet.Tests/WalletTransactionHandlerTest.cs

@@ -620,7 +620,7 @@ public void When_BuildTransactionIsCalled_Then_FeeIsDeductedFromAmountsInTransac
             {
                 AccountReference = walletReference,
                 MinConfirmations = 0,
-                FeeType = FeeType.Low,
+                TransactionFee = Money.Coins(0.0001m),
                 WalletPassword = "password",
                 Recipients = new[]
                 {
@@ -636,6 +636,139 @@ public void When_BuildTransactionIsCalled_Then_FeeIsDeductedFromAmountsInTransac
             Assert.True(transaction.Outputs.Count(i => i.Value.Satoshi < 5_000_000_000) == 2); // 2 outputs should have fees taken from the amount
         }
 
+        [Fact]
+        public void When_BuildTransactionIsCalledWithoutTransactionFee_Then_FeeIsDeductedFromSingleOutputInTransaction()
+        {
+            DataFolder dataFolder = CreateDataFolder(this);
+
+            IWalletRepository walletRepository = new SQLiteWalletRepository(this.LoggerFactory.Object, dataFolder, this.Network, DateTimeProvider.Default, new ScriptAddressReader())
+            {
+                TestMode = true
+            };
+
+            var walletFeePolicy = new Mock<IWalletFeePolicy>();
+            walletFeePolicy.Setup(w => w.GetFeeRate(FeeType.Low.ToConfirmations())).Returns(new FeeRate(20000));
+
+            var walletManager = new WalletManager(this.LoggerFactory.Object, this.Network, new ChainIndexer(this.Network), new WalletSettings(NodeSettings.Default(this.Network)),
+                dataFolder, walletFeePolicy.Object, new Mock<IAsyncProvider>().Object, new NodeLifetime(), DateTimeProvider.Default, this.scriptAddressReader, walletRepository);
+
+            walletManager.Start();
+
+            var reserveUtxoService = new ReserveUtxoService(this.loggerFactory, new Mock<ISignals>().Object);
+
+            var walletTransactionHandler = new WalletTransactionHandler(this.LoggerFactory.Object, walletManager, walletFeePolicy.Object, this.Network, this.standardTransactionPolicy, reserveUtxoService);
+
+            (Wallet wallet, ExtKey extKey) = WalletTestsHelpers.GenerateBlankWalletWithExtKey("myWallet1", "password", walletRepository);
+
+            walletManager.Wallets.Add(wallet);
+
+            int accountIndex = 0;
+            ExtKey addressExtKey = extKey.Derive(new KeyPath($"m/44'/{this.Network.Consensus.CoinType}'/{accountIndex}'"));
+            ExtPubKey extPubKey = addressExtKey.Neuter();
+
+            HdAccount account = wallet.AddNewAccount(extPubKey, accountName: "account1");
+
+            var address = account.ExternalAddresses.First();
+            var destination = account.InternalAddresses.First();
+            var destination2 = account.InternalAddresses.Skip(1).First();
+            var destination3 = account.InternalAddresses.Skip(2).First();
+
+            // Wallet with 4 coinbase outputs of 50 = 200.
+            var chain = new ChainIndexer(wallet.Network);
+            WalletTestsHelpers.AddBlocksWithCoinbaseToChain(wallet.Network, chain, address, 4);
+
+            var walletReference = new WalletAccountReference
+            {
+                AccountName = "account1",
+                WalletName = "myWallet1"
+            };
+
+            // Create a transaction with 3 outputs 50 + 50 + 50 = 150 but with fees charged to recipients.
+            var context = new TransactionBuildContext(this.Network)
+            {
+                AccountReference = walletReference,
+                MinConfirmations = 0,
+                FeeType = FeeType.Low,
+                WalletPassword = "password",
+                Recipients = new[]
+                {
+                    new Recipient { Amount = new Money(50, MoneyUnit.BTC), ScriptPubKey = destination.ScriptPubKey, SubtractFeeFromAmount = true },
+                    new Recipient { Amount = new Money(50, MoneyUnit.BTC), ScriptPubKey = destination2.ScriptPubKey, SubtractFeeFromAmount = false },
+                    new Recipient { Amount = new Money(50, MoneyUnit.BTC), ScriptPubKey = destination3.ScriptPubKey, SubtractFeeFromAmount = false }
+                }.ToList()
+            };
+
+            Transaction transaction = walletTransactionHandler.BuildTransaction(context);
+            Assert.Equal(3, transaction.Inputs.Count); // 3 inputs
+            Assert.Equal(3, transaction.Outputs.Count); // 3 outputs with change
+            Assert.True(transaction.Outputs.Count(i => i.Value.Satoshi < 5_000_000_000) == 1); // 1 output should have fees taken from the amount
+        }
+
+        [Fact]
+        public void When_BuildTransactionIsCalledWithoutTransactionFee_Then_MultipleSubtractFeeRecipients_ThrowsException()
+        {
+            DataFolder dataFolder = CreateDataFolder(this);
+
+            IWalletRepository walletRepository = new SQLiteWalletRepository(this.LoggerFactory.Object, dataFolder, this.Network, DateTimeProvider.Default, new ScriptAddressReader())
+            {
+                TestMode = true
+            };
+
+            var walletFeePolicy = new Mock<IWalletFeePolicy>();
+            walletFeePolicy.Setup(w => w.GetFeeRate(FeeType.Low.ToConfirmations())).Returns(new FeeRate(20000));
+
+            var walletManager = new WalletManager(this.LoggerFactory.Object, this.Network, new ChainIndexer(this.Network), new WalletSettings(NodeSettings.Default(this.Network)),
+                dataFolder, walletFeePolicy.Object, new Mock<IAsyncProvider>().Object, new NodeLifetime(), DateTimeProvider.Default, this.scriptAddressReader, walletRepository);
+
+            walletManager.Start();
+
+            var reserveUtxoService = new ReserveUtxoService(this.loggerFactory, new Mock<ISignals>().Object);
+
+            var walletTransactionHandler = new WalletTransactionHandler(this.LoggerFactory.Object, walletManager, walletFeePolicy.Object, this.Network, this.standardTransactionPolicy, reserveUtxoService);
+
+            (Wallet wallet, ExtKey extKey) = WalletTestsHelpers.GenerateBlankWalletWithExtKey("myWallet1", "password", walletRepository);
+
+            walletManager.Wallets.Add(wallet);
+
+            int accountIndex = 0;
+            ExtKey addressExtKey = extKey.Derive(new KeyPath($"m/44'/{this.Network.Consensus.CoinType}'/{accountIndex}'"));
+            ExtPubKey extPubKey = addressExtKey.Neuter();
+
+            HdAccount account = wallet.AddNewAccount(extPubKey, accountName: "account1");
+
+            var address = account.ExternalAddresses.First();
+            var destination = account.InternalAddresses.First();
+            var destination2 = account.InternalAddresses.Skip(1).First();
+            var destination3 = account.InternalAddresses.Skip(2).First();
+
+            // Wallet with 4 coinbase outputs of 50 = 200.
+            var chain = new ChainIndexer(wallet.Network);
+            WalletTestsHelpers.AddBlocksWithCoinbaseToChain(wallet.Network, chain, address, 4);
+
+            var walletReference = new WalletAccountReference
+            {
+                AccountName = "account1",
+                WalletName = "myWallet1"
+            };
+
+            // Create a transaction with 3 outputs 50 + 50 + 50 = 150 but with fees charged to recipients.
+            var context = new TransactionBuildContext(this.Network)
+            {
+                AccountReference = walletReference,
+                MinConfirmations = 0,
+                FeeType = FeeType.Low,
+                WalletPassword = "password",
+                Recipients = new[]
+                {
+                    new Recipient { Amount = new Money(50, MoneyUnit.BTC), ScriptPubKey = destination.ScriptPubKey, SubtractFeeFromAmount = true },
+                    new Recipient { Amount = new Money(50, MoneyUnit.BTC), ScriptPubKey = destination2.ScriptPubKey, SubtractFeeFromAmount = true },
+                    new Recipient { Amount = new Money(50, MoneyUnit.BTC), ScriptPubKey = destination3.ScriptPubKey, SubtractFeeFromAmount = false }
+                }.ToList()
+            };
+
+            Assert.Throws<WalletException>(() => walletTransactionHandler.BuildTransaction(context));
+        }
+
         public static TransactionBuildContext CreateContext(Network network, WalletAccountReference accountReference, string password,
             Script destinationScript, Money amount, FeeType feeType, int minConfirmations, string opReturnData = null, List<Recipient> recipients = null)
         {

src/Stratis.Bitcoin.Features.Wallet/Controllers/WalletController.cs

@@ -829,6 +829,15 @@ public async Task<IActionResult> OfflineSignRequest([FromBody] OfflineSignReques
             return await this.Execute(request, cancellationToken, async (req, token) => this.Json(await this.walletService.OfflineSignRequest(req, token)));
         }
 
+        [HttpPost]
+        [Route("consolidate")]
+        public async Task<IActionResult> Consolidate([FromBody] ConsolidationRequest request,
+            CancellationToken cancellationToken = default(CancellationToken))
+        {
+            return await this.Execute(request, cancellationToken,
+                async (req, token) => this.Json(await this.walletService.Consolidate(req, token)));
+        }
+
         private TransactionItemModel FindSimilarReceivedTransactionOutput(List<TransactionItemModel> items,
             TransactionData transaction)
         {

src/Stratis.Bitcoin.Features.Wallet/Interfaces/IWalletTransactionHandler.cs

@@ -40,5 +40,7 @@ public interface IWalletTransactionHandler
         /// <param name="context">The context that is used to build a new transaction.</param>
         /// <returns>The estimated fee.</returns>
         Money EstimateFee(TransactionBuildContext context);
+
+        int EstimateSize(TransactionBuildContext context);
     }
 }

src/Stratis.Bitcoin.Features.Wallet/Models/RequestModels.cs

@@ -989,4 +989,43 @@ public class SweepRequest : RequestModel
 
         public bool Broadcast { get; set; }
     }
+
+    public sealed class ConsolidationRequest : RequestModel
+    {
+        public ConsolidationRequest()
+        {
+            this.AccountName = WalletManager.DefaultAccount;
+        }
+
+        [Required(ErrorMessage = "The name of the wallet is missing.")]
+        public string WalletName { get; set; }
+
+        /// <summary>
+        /// The account from which UTXOs should be consolidated.
+        /// If this is not set the default account of the selected wallet will be used.
+        /// </summary>
+        public string AccountName { get; set; }
+
+        [Required(ErrorMessage = "A password is required.")]
+        public string WalletPassword { get; set; }
+
+        /// <summary>
+        /// If this is set, only UTXOs within this wallet address will be consolidated.
+        /// If it is not set, all the UTXOs within the selected account will be consolidated.
+        /// </summary>
+        public string SingleAddress { get; set; }
+
+        /// <summary>
+        /// Which address the UTXOs should be sent to. It does not have to be within the wallet.
+        /// If it is not provided the UTXOs will be consolidated to an unused address within the specified wallet.
+        /// </summary>
+        public string DestinationAddress { get; set; }
+
+        /// <summary>
+        /// If provided, UTXOs that are larger in value will not be consolidated.
+        /// Dust UTXOs will not be consolidated regardless of their value, so there is an implicit lower bound as well.
+        /// </summary>
+        [MoneyFormat(isRequired: false, ErrorMessage = "The amount is not in the correct format.")]
+        public string UtxoValueThreshold { get; set; }
+    }
 }

src/Stratis.Bitcoin.Features.Wallet/Services/IWalletService.cs

@@ -59,5 +59,8 @@ Task<BuildOfflineSignResponse> BuildOfflineSignRequest(BuildOfflineSignRequest r
 
         Task<WalletBuildTransactionModel> OfflineSignRequest(OfflineSignRequest request,
             CancellationToken cancellationToken);
+
+        Task<string> Consolidate(ConsolidationRequest request,
+            CancellationToken cancellationToken);
     }
 }

src/Stratis.Bitcoin.Features.Wallet/Services/WalletService.cs

@@ -1382,6 +1382,101 @@ public async Task<WalletBuildTransactionModel> OfflineSignRequest(OfflineSignReq
             }, cancellationToken);
         }
 
+        public async Task<string> Consolidate(ConsolidationRequest request, CancellationToken cancellationToken)
+        {
+            return await Task.Run(() =>
+            {
+                var utxos = new List<UnspentOutputReference>();
+                var accountReference = new WalletAccountReference(request.WalletName, request.AccountName);
+
+                if (!string.IsNullOrWhiteSpace(request.SingleAddress))
+                {
+                    utxos = this.walletManager.GetSpendableTransactionsInWallet(request.WalletName).Where(u => u.Address.Address == request.SingleAddress || u.Address.Address == request.SingleAddress).OrderBy(u2 => u2.Transaction.Amount).ToList();
+                }
+                else
+                {
+                    utxos = this.walletManager.GetSpendableTransactionsInAccount(accountReference).OrderBy(u2 => u2.Transaction.Amount).ToList();
+                }
+
+                if (utxos.Count == 0)
+                {
+                    throw new FeatureException(HttpStatusCode.BadRequest, "Failed to locate any unspent outputs to consolidate.",
+                        "Failed to locate any unspent outputs to consolidate.");
+                }
+
+                if (utxos.Count == 1)
+                {
+                    throw new FeatureException(HttpStatusCode.BadRequest, "Already consolidated.",
+                        "Already consolidated.");
+                }
+
+
+                if (!string.IsNullOrWhiteSpace(request.UtxoValueThreshold))
+                {
+                    var threshold = Money.Parse(request.UtxoValueThreshold);
+
+                    utxos = utxos.Where(u => u.Transaction.Amount <= threshold).ToList();
+                }
+
+                Script destination;
+                if (!string.IsNullOrWhiteSpace(request.DestinationAddress))
+                {
+                    destination = BitcoinAddress.Create(request.DestinationAddress, this.network).ScriptPubKey;
+                }
+                else
+                {
+                    destination = this.walletManager.GetUnusedAddress(accountReference).ScriptPubKey;
+                }
+
+                Money totalToSend = Money.Zero;
+                var outpoints = new List<OutPoint>();
+
+                TransactionBuildContext context = null;
+
+                foreach (var utxo in utxos)
+                {
+                    totalToSend += utxo.Transaction.Amount;
+                    outpoints.Add(utxo.ToOutPoint());
+                    
+                    context = new TransactionBuildContext(this.network)
+                    {
+                        AccountReference = accountReference,
+                        AllowOtherInputs = false,
+                        FeeType = FeeType.Medium,
+                        // It is intended that consolidation should result in no change address, so the fee has to be subtracted from the single recipient.
+                        Recipients = new List<Recipient>() { new Recipient() { ScriptPubKey = destination, Amount = totalToSend, SubtractFeeFromAmount = true } },
+                        SelectedInputs = outpoints,
+
+                        Sign = false
+                    };
+
+                    // Note that this is the virtual size taking the witness scale factor of the current network into account, and not the raw byte count.
+                    int size = this.walletTransactionHandler.EstimateSize(context);
+
+                    // Leave a bit of an error margin for size estimates that are not completely correct.
+                    if (size > (0.95m * this.network.Consensus.Options.MaxStandardTxWeight))
+                        break;
+                }
+                
+                // Build the final version of the consolidation transaction.
+                context = new TransactionBuildContext(this.network)
+                {
+                    AccountReference = accountReference,
+                    AllowOtherInputs = false,
+                    FeeType = FeeType.Medium,
+                    Recipients = new List<Recipient>() { new Recipient() { ScriptPubKey = destination, Amount = totalToSend, SubtractFeeFromAmount = true } },
+                    SelectedInputs = outpoints,
+                    WalletPassword = request.WalletPassword,
+
+                    Sign = true
+                };
+
+                Transaction transaction = this.walletTransactionHandler.BuildTransaction(context);
+
+                return transaction.ToHex();
+            }, cancellationToken);
+        }
+
         private TransactionItemModel FindSimilarReceivedTransactionOutput(List<TransactionItemModel> items,
             TransactionData transaction)
         {