Wallet moved to litedb (#179)

* Adding a wallet store based on litedb

* Fix all build errors apart for tests

* Wallet uses litedb fully

* Wallet db file per wallet

* Fix more tests and generate test chains

* Fix some tests

* Fix some more tests

* Fix some more tests

* Solution compiles

* Wallet manager tests passing

* Wallet tip in store and fix tests

* Fixing more tests

* All tests pass

* Upgrade litedb - all tests pass

* Fix integration tests

* Improve wallet perf

* move wallet db to its own folder

* move wallet to its own db

* Rename TransactionData to TransactionOutputData

* Rename wallet files

* Fix tests

* Check for consumers on broadcaster

* Add consumer count to event consumer interface

* uniform wallet value format

* Fix wallet backup bug

* Cold stake page optimization

* Fix bug

* fix calculation error on balance

* delete unused method totalUnconfirmedBalance

* More optimization and add account index to trx store

* Create litedb queries to optimize balance checks

* Only calculate spendable amounts when its used

Co-authored-by: Hunter <thecryptohunter@protonmail.com>

Author: dangershony

src/Blockcore/Blockcore.csproj

@@ -17,7 +17,7 @@
   <ItemGroup>
     <PackageReference Include="ConcurrentHashSet" Version="1.1.0" />
     <PackageReference Include="DBreeze" Version="1.95.0" />
-    <PackageReference Include="LiteDB" Version="4.1.4" />
+    <PackageReference Include="LiteDB" Version="5.0.8" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning" Version="4.1.1" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="3.1.4" />
     <PackageReference Include="NLog" Version="4.7.0" />

src/Blockcore/Broadcasters/BroadcasterBase.cs

@@ -40,14 +40,17 @@ public void Init(ClientEventBroadcasterSettings broadcasterSettings)
                 $"Broadcast {this.GetType().Name}",
                 async token =>
                 {
-                    foreach (EventBase clientEvent in this.GetMessages())
+                    if (this.eventsHub.HasConsumers)
                     {
-                        this.eventsHub.OnEvent(clientEvent);
+                        foreach (EventBase clientEvent in this.GetMessages())
+                        {
+                            this.eventsHub.OnEvent(clientEvent);
+                        }
                     }
                 },
                 this.nodeLifetime.ApplicationStopping,
                 repeatEvery: TimeSpan.FromSeconds(Math.Max(broadcasterSettings.BroadcastFrequencySeconds, 5)),
-                startAfter: TimeSpans.FiveSeconds);
+                startAfter: TimeSpans.TenSeconds);
         }
 
         protected abstract IEnumerable<EventBase> GetMessages();

src/Blockcore/Broadcasters/IEventsSubscriptionService.cs

@@ -9,6 +9,8 @@ public interface IEventsSubscriptionService
     {
         void Init();
 
+        bool HasConsumers { get; }
+
         void Subscribe(string id, string name);
 
         void Unsubscribe(string id, string name);

src/Blockcore/Configuration/DataFolder.cs

@@ -33,6 +33,7 @@ public DataFolder(string path)
             this.WalletPath = Path.Combine(path);
             this.LogPath = Path.Combine(path, "logs");
             this.ApplicationsPath = Path.Combine(path, "apps");
+            this.WalletFolderPath = Path.Combine(path, "txdb");
             this.DnsMasterFilePath = path;
             this.SmartContractStatePath = Path.Combine(path, "contracts");
             this.ProvenBlockHeaderPath = Path.Combine(path, "provenheaders");
@@ -77,6 +78,9 @@ public DataFolder(string path)
         /// <seealso cref="Features.Wallet.WalletManager.LoadWallet"/>
         public string WalletPath { get; internal set; }
 
+        /// <summary>Path to wallet folder.</summary>
+        public string WalletFolderPath { get; internal set; }
+
         /// <summary>Path to log files.</summary>
         /// <seealso cref="Logging.LoggingConfiguration"/>
         public string LogPath { get; internal set; }

src/Blockcore/Utilities/FileStorage.cs

@@ -193,6 +193,43 @@ public T LoadByFileName(string fileName)
             return JsonConvert.DeserializeObject<T>(File.ReadAllText(filePath));
         }
 
+        /// <summary>
+        /// Creates a copy of legacy wallets in to v2 wallets.
+        /// </summary>
+        /// <param name="fileExtension">The extension.</param>
+        /// <returns>A list of objects of type <see cref="T"/> whose persisted files have the specified extension. </returns>
+        public IEnumerable<T> CloneLegacyWallet(string fileExtension)
+        {
+            Guard.NotEmpty(fileExtension, nameof(fileExtension));
+
+            // Get the paths of files with the extension
+            IEnumerable<string> filesPaths = this.GetFilesPaths(fileExtension);
+
+            var files = new List<T>();
+            foreach (string filePath in filesPaths)
+            {
+                if (File.ReadAllText(filePath).Contains("blockLocator"))
+                {
+                    // This is an old legacy wallet, back it up and migrate.
+
+                    string fileName = Path.GetFileName(filePath);
+                    string newFileName = fileName.Replace(fileExtension, $"{fileExtension}.backup");
+                    string newFilePath = Path.Combine(this.FolderPath, newFileName);
+
+                    if (!this.Exists(newFilePath))
+                    {
+                        File.Copy(filePath, newFilePath);
+
+                        // Upgrade the wallet
+                        T loadedFile = this.LoadByFileName(fileName);
+                        this.SaveToFile(loadedFile, fileName);
+                    }
+                }
+            }
+
+            return files;
+        }
+
         /// <summary>
         /// Loads all the objects that have file with the specified extension.
         /// </summary>
@@ -218,4 +255,4 @@ public IEnumerable<T> LoadByFileExtension(string fileExtension)
             return files;
         }
     }
-}
+}

src/Blockcore/Utilities/HashHeightPair.cs

@@ -1,4 +1,5 @@
-using NBitcoin;
+using System;
+using NBitcoin;
 
 namespace Blockcore.Utilities
 {
@@ -94,5 +95,44 @@ public override int GetHashCode()
         {
             return base.GetHashCode();
         }
+
+        public static bool TryParse(string str, out HashHeightPair result)
+        {
+            Guard.NotEmpty(str, nameof(str));
+
+            result = null;
+
+            string[] splitted = str.Split('-');
+            if (splitted.Length != 2)
+            {
+                return false;
+            }
+
+            if (!int.TryParse(splitted[0], out int index))
+            {
+                return false;
+            }
+
+            if (!uint256.TryParse(splitted[1], out uint256 hash))
+            {
+                return false;
+            }
+
+            result = new HashHeightPair(hash, index);
+
+            return true;
+        }
+
+        public static HashHeightPair Parse(string str)
+        {
+            HashHeightPair result;
+
+            if (TryParse(str, out result))
+            {
+                return result;
+            }
+
+            throw new FormatException("The format of the outpoint is incorrect");
+        }
     }
 }

src/Blockcore/Utilities/JsonConverters/OutpointJsonConverter.cs

@@ -0,0 +1,46 @@
+using System;
+using System.IO;
+using NBitcoin;
+using Newtonsoft.Json;
+
+namespace Blockcore.Utilities.JsonConverters
+{
+    /// <summary>
+    /// Converter used to convert a <see cref="OutPoint"/> to and from JSON.
+    /// </summary>
+    /// <seealso cref="Newtonsoft.Json.JsonConverter" />
+    public class OutPointJsonConverter : JsonConverter
+    {
+        /// <inheritdoc />
+        public override bool CanConvert(Type objectType)
+        {
+            return typeof(OutPoint) == objectType;
+        }
+
+        /// <inheritdoc />
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            if (reader.TokenType == JsonToken.Null)
+                return null;
+
+            try
+            {
+                return OutPoint.Parse((string)reader.Value);
+            }
+            catch (EndOfStreamException)
+            {
+            }
+            catch (FormatException)
+            {
+            }
+
+            throw new JsonObjectException("Invalid bitcoin object of type " + objectType.Name, reader);
+        }
+
+        /// <inheritdoc />
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            writer.WriteValue(value.ToString());
+        }
+    }
+}

src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs

@@ -13,7 +13,7 @@ public class AddressIndexRepository : MemorySizeCache<string, AddressIndexerData
     {
         private const string DbAddressDataKey = "AddrData";
 
-        private readonly LiteCollection<AddressIndexerData> addressIndexerDataCollection;
+        private readonly ILiteCollection<AddressIndexerData> addressIndexerDataCollection;
 
         private readonly ILogger logger;
 
@@ -58,7 +58,7 @@ public List<string> GetAddressesHigherThanHeight(int height)
             this.SaveAllItems();
 
             // Need to specify index name explicitly so that it gets used for the query.
-            IEnumerable<AddressIndexerData> affectedAddresses = this.addressIndexerDataCollection.Find(Query.GT("BalanceChangedHeightIndex", height));
+            IEnumerable<AddressIndexerData> affectedAddresses = this.addressIndexerDataCollection.Find(a => a.BalanceChanges.Select(ab => ab.BalanceChangedHeight).Any(ab => ab > height));
 
             // Per LiteDb documentation:
             // "Returning an IEnumerable your code still connected to datafile.

src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexer.cs

@@ -18,7 +18,6 @@
 using LiteDB;
 using Microsoft.Extensions.Logging;
 using NBitcoin;
-using FileMode = LiteDB.FileMode;
 using Script = NBitcoin.Script;
 
 namespace Blockcore.Features.BlockStore.AddressIndexing
@@ -83,7 +82,7 @@ public class AddressIndexer : IAddressIndexer
 
         private LiteDatabase db;
 
-        private LiteCollection<AddressIndexerTipData> tipDataStore;
+        private ILiteCollection<AddressIndexerTipData> tipDataStore;
 
         /// <summary>A mapping between addresses and their balance changes.</summary>
         /// <remarks>All access should be protected by <see cref="lockObject"/>.</remarks>
@@ -176,8 +175,7 @@ public void Initialize()
 
             string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename);
 
-            FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
-            this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
+            this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true });
 
             this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory);
 
@@ -656,4 +654,4 @@ public void Dispose()
             this.db?.Dispose();
         }
     }
-}
+}

src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs

@@ -17,11 +17,11 @@ public sealed class AddressIndexerOutpointsRepository : MemoryCache<string, OutP
 
         /// <summary>Represents the output collection.</summary>
         /// <remarks>Should be protected by <see cref="LockObject"/></remarks>
-        private readonly LiteCollection<OutPointData> addressIndexerOutPointData;
+        private readonly ILiteCollection<OutPointData> addressIndexerOutPointData;
 
         /// <summary>Represents the rewind data collection.</summary>
         /// <remarks>Should be protected by <see cref="LockObject"/></remarks>
-        private readonly LiteCollection<AddressIndexerRewindData> addressIndexerRewindData;
+        private readonly ILiteCollection<AddressIndexerRewindData> addressIndexerRewindData;
 
         private readonly ILogger logger;
 
@@ -158,4 +158,4 @@ protected override bool IsCacheFullLocked(CacheItem item)
             return this.totalSize + 1 > this.maxCacheItems;
         }
     }
-}
+}