Port of stratx implementation of sweep (#236)

Author: xandronus

src/Features/Blockcore.Features.BlockStore/Api/Controllers/BlockStoreController.cs

@@ -1,8 +1,10 @@
 using System;
+using System.Collections.Generic;
 using System.Net;
 using Blockcore.Base;
 using Blockcore.Consensus.BlockInfo;
 using Blockcore.Consensus.Chain;
+using Blockcore.Consensus.TransactionInfo;
 using Blockcore.Controllers.Models;
 using Blockcore.Features.BlockStore.AddressIndexing;
 using Blockcore.Features.BlockStore.Api.Models;
@@ -44,24 +46,30 @@ public class BlockStoreController : Controller
         /// <summary>Current network for the active controller instance.</summary>
         private readonly Network network;
 
+        /// <summary>UTXO indexer.</summary>
+        private readonly IUtxoIndexer utxoIndexer;
+
         public BlockStoreController(
             Network network,
             ILoggerFactory loggerFactory,
             IBlockStore blockStore,
             IChainState chainState,
             ChainIndexer chainIndexer,
-            IAddressIndexer addressIndexer)
+            IAddressIndexer addressIndexer,
+            IUtxoIndexer utxoIndexer)
         {
             Guard.NotNull(network, nameof(network));
             Guard.NotNull(loggerFactory, nameof(loggerFactory));
             Guard.NotNull(chainState, nameof(chainState));
             Guard.NotNull(addressIndexer, nameof(addressIndexer));
+            Guard.NotNull(utxoIndexer, nameof(utxoIndexer));
 
             this.addressIndexer = addressIndexer;
             this.network = network;
             this.blockStore = blockStore;
             this.chainState = chainState;
             this.chainIndexer = chainIndexer;
+            this.utxoIndexer = utxoIndexer;
             this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
         }
 
@@ -210,5 +218,39 @@ public IActionResult GetVerboseAddressesBalancesData(string addresses)
                 return ErrorHelpers.BuildErrorResponse(HttpStatusCode.BadRequest, e.Message, e.ToString());
             }
         }
+
+        /// <summary>Returns every UTXO as of a given block height. This may take some time for large chains.</summary>
+        /// <param name="atBlockHeight">Only process blocks up to this height for the purposes of constructing the UTXO set.</param>
+        /// <returns>A result object containing the UTXOs.</returns>
+        /// <response code="200">Returns the UTXO set.</response>
+        /// <response code="400">Unexpected exception occurred</response>
+        [Route(BlockStoreRouteEndPoint.GetUtxoSet)]
+        [HttpGet]
+        [ProducesResponseType((int)HttpStatusCode.OK)]
+        [ProducesResponseType((int)HttpStatusCode.BadRequest)]
+        public IActionResult GetUtxoSet(int atBlockHeight)
+        {
+            try
+            {
+                ReconstructedCoinviewContext coinView = this.utxoIndexer.GetCoinviewAtHeight(atBlockHeight);
+
+                var outputs = new List<UtxoModel>();
+
+                foreach (OutPoint outPoint in coinView.UnspentOutputs)
+                {
+                    TxOut txOut = coinView.Transactions[outPoint.Hash].Outputs[outPoint.N];
+                    var utxo = new UtxoModel() { TxId = outPoint.Hash, Index = outPoint.N, ScriptPubKey = txOut.ScriptPubKey, Value = txOut.Value };
+
+                    outputs.Add(utxo);
+                }
+
+                return this.Json(outputs);
+            }
+            catch (Exception e)
+            {
+                this.logger.LogError("Exception occurred: {0}", e.ToString());
+                return ErrorHelpers.BuildErrorResponse(HttpStatusCode.BadRequest, e.Message, e.ToString());
+            }
+        }
     }
 }

src/Features/Blockcore.Features.BlockStore/Api/Models/BlockStoreRouteEndPoint.cs

@@ -7,5 +7,6 @@ public static class BlockStoreRouteEndPoint
         public const string GetAddressIndexerTip = "addressindexertip";
         public const string GetBlock = "block";
         public const string GetBlockCount = "GetBlockCount";
+        public const string GetUtxoSet = "getutxoset";
     }
 }

src/Features/Blockcore.Features.BlockStore/Api/Models/UtxoModel.cs

@@ -0,0 +1,21 @@
+using Blockcore.Consensus.ScriptInfo;
+using NBitcoin;
+using Newtonsoft.Json;
+
+namespace Blockcore.Features.BlockStore.Models
+{
+    public class UtxoModel
+    {
+        [JsonProperty]
+        public uint256 TxId { get; set; }
+
+        [JsonProperty]
+        public uint Index { get; set; }
+
+        [JsonProperty]
+        public Script ScriptPubKey { get; set; }
+
+        [JsonProperty]
+        public Money Value { get; set; }
+    }
+}

src/Features/Blockcore.Features.BlockStore/BlockStoreFeature.cs

@@ -209,6 +209,7 @@ public static IFullNodeBuilder UseBlockStore(this IFullNodeBuilder fullNodeBuild
                         services.AddSingleton<StoreSettings>();
                         services.AddSingleton<IBlockStoreQueueFlushCondition, BlockStoreQueueFlushCondition>();
                         services.AddSingleton<IAddressIndexer, AddressIndexer>();
+                        services.AddSingleton<IUtxoIndexer, UtxoIndexer>();
 
                         services.AddSingleton<IPrunedBlockRepository, PrunedBlockRepository>();
                         services.AddSingleton<IPruneBlockStoreService, PruneBlockStoreService>();

src/Features/Blockcore.Features.BlockStore/UtxoIndexing/ReconstructedCoinviewContext.cs

@@ -0,0 +1,28 @@
+using System.Collections.Generic;
+using Blockcore.Consensus.TransactionInfo;
+using NBitcoin;
+
+namespace Blockcore.Features.BlockStore
+{
+    /// <summary>
+    /// This is so named to indicate that it is not really intended for use outside of the block store's controller, i.e. it is not part of consensus.
+    /// </summary>
+    public class ReconstructedCoinviewContext
+    {
+        /// <summary>
+        /// All of the outputs that haven't been spent at this point in time.
+        /// </summary>
+        public HashSet<OutPoint> UnspentOutputs { get; }
+
+        /// <summary>
+        /// Easy access to all of the loaded transactions.
+        /// </summary>
+        public Dictionary<uint256, Transaction> Transactions { get; }
+
+        public ReconstructedCoinviewContext()
+        {
+            this.UnspentOutputs = new HashSet<OutPoint>();
+            this.Transactions = new Dictionary<uint256, Transaction>();
+        }
+    }
+}

src/Features/Blockcore.Features.BlockStore/UtxoIndexing/UtxoIndexer.cs

@@ -0,0 +1,119 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+using NBitcoin;
+using Blockcore.Interfaces;
+using Blockcore.Networks;
+using Blockcore.Consensus.Chain;
+using Blockcore.Consensus.BlockInfo;
+using Blockcore.Consensus.TransactionInfo;
+
+namespace Blockcore.Features.BlockStore
+{
+    public interface IUtxoIndexer
+    {
+        ReconstructedCoinviewContext GetCoinviewAtHeight(int blockHeight);
+    }
+
+    /// <summary>
+    /// This is a separate use case from the address indexer, as we need full details about the current items in the UTXO set in order to
+    /// be able to potentially build transactions from them. The address indexer is also intended for long-running use cases, and does
+    /// incur overhead in processing each block. This indexer, conversely, is designed for occasional usage without the requirement
+    /// of persisting its output for rapid lookup.
+    /// The actual coindb is not well suited to this kind of query either, as it is being accessed frequently and lacks a 'snapshot'
+    /// facility. This indexer therefore trades query speed for implementation simplicity, as we only have to enumerate the entire chain
+    /// without regard for significantly impacting other components.
+    /// Allowing the height to be specified is an additional convenience that the coindb on its own would not give us without additional
+    /// consistency safeguards, which may be invasive. For example, halting consensus at a specific height to build a snapshot.
+    /// </summary>
+    /// <remarks>This borrows heavily from the <see cref="Stratis.Bitcoin.Features.SmartContracts.ReflectionExecutor.Controllers.BalancesController"/>, but is less specialised.</remarks>
+    public class UtxoIndexer : IUtxoIndexer
+    {
+        private readonly Network network;
+        private readonly ChainIndexer chainIndexer;
+        private readonly IBlockStore blockStore;
+        private readonly ILogger logger;
+
+        public UtxoIndexer(Network network, ChainIndexer chainIndexer, IBlockStore blockStore, ILoggerFactory loggerFactory)
+        {
+            this.network = network;
+            this.chainIndexer = chainIndexer;
+            this.blockStore = blockStore;
+            this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
+        }
+
+        public ReconstructedCoinviewContext GetCoinviewAtHeight(int blockHeight)
+        {
+            var coinView = new ReconstructedCoinviewContext();
+
+            // TODO: Make this a command line option
+            const int batchSize = 1000;
+
+            IEnumerable<ChainedHeader> allBlockHeaders = this.chainIndexer.EnumerateToTip(this.chainIndexer.Genesis);
+
+            int totalBlocksCounted = 0;
+
+            int i = 0;
+
+            while (true)
+            {
+                List<ChainedHeader> headers = allBlockHeaders.Skip(i * batchSize).Take(batchSize).ToList();
+                List<Block> blocks = this.blockStore.GetBlocks(headers.Select(x => x.HashBlock).ToList());
+
+                foreach (Block block in blocks)
+                {
+                    this.AdjustCoinviewForBlock(block, coinView);
+                    totalBlocksCounted += 1;
+
+                    // We have reached the block height asked for.
+                    if (totalBlocksCounted >= blockHeight)
+                        break;
+                }
+
+                // We have seen every block up to the tip or the chosen block height.
+                if (headers.Count < batchSize || totalBlocksCounted >= blockHeight)
+                    break;
+
+                // Give the user some feedback every 10k blocks
+                if (i % 10 == 9)
+                    this.logger.LogInformation($"Processed {totalBlocksCounted} blocks for UTXO indexing.");
+
+                i++;
+            }
+
+            return coinView;
+        }
+
+        private void AdjustCoinviewForBlock(Block block, ReconstructedCoinviewContext coinView)
+        {
+            foreach (Transaction tx in block.Transactions)
+            {
+                // Add outputs 
+                for (int i = 0; i < tx.Outputs.Count; i++)
+                {
+                    TxOut output = tx.Outputs[i];
+
+                    if (output.Value <= 0 || output.ScriptPubKey.IsUnspendable)
+                        continue;
+
+                    coinView.UnspentOutputs.Add(new OutPoint(tx, i));
+                    coinView.Transactions[tx.GetHash()] = tx;
+                }
+
+                // Spend inputs. Coinbases are special in that they don't reference previous transactions for their inputs, so ignore them.
+                if (tx.IsCoinBase)
+                    continue;
+
+                foreach (TxIn txIn in tx.Inputs)
+                {
+                    // TODO: This is inefficient because the Transactions dictionary entry does not get deleted, so extra memory gets used
+                    if (!coinView.UnspentOutputs.Remove(txIn.PrevOut))
+                    {
+                        throw new Exception("Attempted to spend a prevOut that isn't in the coinview at this time.");
+                    }
+                }
+            }
+        }
+    }
+}

src/Features/Blockcore.Features.Wallet/Api/Controllers/WalletController.cs

@@ -1569,6 +1569,27 @@ public IActionResult DistributeUtxos([FromBody] DistributeUtxosRequest request)
             }
         }
 
+        /// <summary>
+        /// Sweeps one or more private keys to another address.
+        /// </summary>
+        /// <param name="request">request</param>
+        /// <returns>List of transactions</returns>
+        [HttpPost]
+        [Route("sweep")]
+        public IActionResult Sweep([FromBody] SweepRequest request)
+        {
+            try
+            {
+                var responseModel = this.walletManager.Sweep(request.PrivateKeys, request.DestinationAddress, request.Broadcast);
+                return this.Json(responseModel);
+            }
+            catch(Exception e)
+            {
+                this.logger.LogError("Exception occurred: {0}", e.ToString());
+                return ErrorHelpers.BuildErrorResponse(HttpStatusCode.BadRequest, e.Message, e.ToString());
+            }
+        }
+
         private void SyncFromBestHeightForRecoveredWallets(DateTime walletCreationDate)
         {
             // After recovery the wallet needs to be synced.

src/Features/Blockcore.Features.Wallet/Api/Models/RequestModels.cs

@@ -920,4 +920,15 @@ public class VerifyRequest : RequestModel
         [Required(ErrorMessage = "A message is required.")]
         public string Message { get; set; }
     }
+
+    public class SweepRequest : RequestModel
+    {
+        [Required(ErrorMessage = "One or more private keys is required.")]
+        public List<string> PrivateKeys { get; set; }
+
+        [Required(ErrorMessage = "A destination address is required.")]
+        public string DestinationAddress { get; set; }
+
+        public bool Broadcast { get; set; }
+    }
 }

src/Features/Blockcore.Features.Wallet/Interfaces/IWalletManager.cs

@@ -420,5 +420,14 @@ public interface IWalletManager
         /// <param name="fromDate">The date after which the transactions should be removed.</param>
         /// <returns>A list of objects made up of a transactions ID along with the time at which they were created.</returns>
         HashSet<(uint256, DateTimeOffset)> RemoveTransactionsFromDate(string walletName, DateTimeOffset fromDate);
+
+        /// <summary>
+        /// Sweeps the funds from the private keys to the destination address.
+        /// </summary>
+        /// <param name="privateKeys">Private keys to sweep funds from in wif format.</param>
+        /// <param name="destAddress">Destination address to sweep funds to.</param>
+        /// <param name="broadcast">Broadcast the transaction to the network.</param>
+        /// <returns>List of sweep transactions.</returns>
+        IEnumerable<string> Sweep(IEnumerable<string> privateKeys, string destAddress, bool broadcast);
     }
 }