Changed LiteDB to v4 for stability. (#242)

* Changed LiteDB to v4 for stability.

* Fixed the grouping for the wallet account and history

* Generate test data

* Regenerated test data for wallet with funds.

* Updated with transactions in wallet with funds.

* Update launchSettings.json

* Add fix to handle downgrade of databases

- Rename the database file name for addressindexer. This means users must manually delete the old file to clear up disk space.
- Attempt to read the wallet database and if it fails, attempt to rename it away and try again (only one time).

* Remove the zero check

- The "affectedAddresses" query fails and must be updated to work properly with LiteDB V4.

* Fix issue with query against LiteDB

- This fixes a crash that occured due to LINQ resulting in an invalid query against V4 of LiteDB. Reverted back to old syntax.

* Merge some old code into AddressIndexer

* Restore the Address, HotAddress and ColdAddress support

* Improve the purge performance

- Improves the purge performance greatly by using query to perform a bulk delete.

Co-authored-by: dangershony <dan.gershony@gmail.com>
Co-authored-by: SondreB <sondre@outlook.com>

Author: 3 people

src/Blockcore/Blockcore.csproj

@@ -24,7 +24,7 @@
   <ItemGroup>
     <PackageReference Include="ConcurrentHashSet" Version="1.1.0" />
     <PackageReference Include="DBreeze" Version="1.97.0" />
-    <PackageReference Include="LiteDB" Version="5.0.9" />
+    <PackageReference Include="LiteDB" Version="4.1.4" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.Versioning" Version="4.2.0" />
     <PackageReference Include="Microsoft.Extensions.Http" Version="5.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />

src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs

@@ -13,7 +13,7 @@ public class AddressIndexRepository : MemorySizeCache<string, AddressIndexerData
     {
         private const string DbAddressDataKey = "AddrData";
 
-        private readonly ILiteCollection<AddressIndexerData> addressIndexerDataCollection;
+        private readonly LiteCollection<AddressIndexerData> addressIndexerDataCollection;
 
         private readonly ILogger logger;
 
@@ -58,7 +58,7 @@ public List<string> GetAddressesHigherThanHeight(int height)
             this.SaveAllItems();
 
             // Need to specify index name explicitly so that it gets used for the query.
-            IEnumerable<AddressIndexerData> affectedAddresses = this.addressIndexerDataCollection.Find(a => a.BalanceChanges.Select(ab => ab.BalanceChangedHeight).Any(ab => ab > height));
+            IEnumerable<AddressIndexerData> affectedAddresses = this.addressIndexerDataCollection.Find(Query.GT("BalanceChangedHeightIndex", height));
 
             // Per LiteDb documentation:
             // "Returning an IEnumerable your code still connected to datafile.

src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexer.cs

@@ -19,6 +19,7 @@
 using Blockcore.Networks;
 using Blockcore.Utilities;
 using LiteDB;
+using FileMode = LiteDB.FileMode;
 using Microsoft.Extensions.Logging;
 using NBitcoin;
 using Script = Blockcore.Consensus.ScriptInfo.Script;
@@ -85,7 +86,7 @@ public class AddressIndexer : IAddressIndexer
 
         private LiteDatabase db;
 
-        private ILiteCollection<AddressIndexerTipData> tipDataStore;
+        private LiteCollection<AddressIndexerTipData> tipDataStore;
 
         /// <summary>A mapping between addresses and their balance changes.</summary>
         /// <remarks>All access should be protected by <see cref="lockObject"/>.</remarks>
@@ -178,7 +179,8 @@ public void Initialize()
 
             string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename);
 
-            this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Upgrade = true });
+            FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared;
+            this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode });
 
             this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory);
 
@@ -365,7 +367,9 @@ private void SaveAll()
                 AddressIndexerTipData tipData = this.tipDataStore.FindAll().FirstOrDefault();
 
                 if (tipData == null)
+                { 
                     tipData = new AddressIndexerTipData();
+                }
 
                 tipData.Height = this.IndexerTip.Height;
                 tipData.TipHashBytes = this.IndexerTip.HashBlock.ToBytes();
@@ -508,7 +512,9 @@ private bool ProcessBlock(Block block, ChainedHeader header)
 
                 // Remove outpoints that were consumed.
                 foreach (OutPoint consumedOutPoint in inputs.Select(x => x.PrevOut))
+                { 
                     this.outpointsRepository.RemoveOutPointData(consumedOutPoint);
+                }
             }
 
             return true;

src/Features/Blockcore.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs

@@ -18,18 +18,21 @@ public sealed class AddressIndexerOutpointsRepository : MemoryCache<string, OutP
 
         /// <summary>Represents the output collection.</summary>
         /// <remarks>Should be protected by <see cref="LockObject"/></remarks>
-        private readonly ILiteCollection<OutPointData> addressIndexerOutPointData;
+        private readonly LiteCollection<OutPointData> addressIndexerOutPointData;
 
         /// <summary>Represents the rewind data collection.</summary>
         /// <remarks>Should be protected by <see cref="LockObject"/></remarks>
-        private readonly ILiteCollection<AddressIndexerRewindData> addressIndexerRewindData;
+        private readonly LiteCollection<AddressIndexerRewindData> addressIndexerRewindData;
 
         private readonly ILogger logger;
 
         private readonly int maxCacheItems;
 
+        private readonly LiteDatabase db;
+
         public AddressIndexerOutpointsRepository(LiteDatabase db, ILoggerFactory loggerFactory, int maxItems = 60_000)
         {
+            this.db = db;
             this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
             this.addressIndexerOutPointData = db.GetCollection<OutPointData>(DbOutputsDataKey);
             this.addressIndexerRewindData = db.GetCollection<AddressIndexerRewindData>(DbOutputsRewindDataKey);
@@ -67,7 +70,9 @@ protected override void ItemRemovedLocked(CacheItem item)
             base.ItemRemovedLocked(item);
 
             if (item.Dirty)
+            {
                 this.addressIndexerOutPointData.Upsert(item.Value);
+            }
         }
 
         public bool TryGetOutPointData(OutPoint outPoint, out OutPointData outPointData)
@@ -93,63 +98,51 @@ public bool TryGetOutPointData(OutPoint outPoint, out OutPointData outPointData)
 
         public void SaveAllItems()
         {
-            lock (this.LockObject)
-            {
-                CacheItem[] dirtyItems = this.Keys.Where(x => x.Dirty).ToArray();
-                this.addressIndexerOutPointData.Upsert(dirtyItems.Select(x => x.Value));
+            CacheItem[] dirtyItems = this.Keys.Where(x => x.Dirty).ToArray();
+
+            this.addressIndexerOutPointData.Upsert(dirtyItems.Select(x => x.Value));
 
-                foreach (CacheItem dirtyItem in dirtyItems)
-                    dirtyItem.Dirty = false;
+            foreach (CacheItem dirtyItem in dirtyItems)
+            {
+                dirtyItem.Dirty = false;
             }
         }
 
         /// <summary>Persists rewind data into the repository.</summary>
         /// <param name="rewindData">The data to be persisted.</param>
         public void RecordRewindData(AddressIndexerRewindData rewindData)
         {
-            lock (this.LockObject)
-            {
-                this.addressIndexerRewindData.Upsert(rewindData);
-            }
+            this.addressIndexerRewindData.Upsert(rewindData);
         }
 
         /// <summary>Deletes rewind data items that were originated at height lower than <paramref name="height"/>.</summary>
         /// <param name="height">The threshold below which data will be deleted.</param>
         public void PurgeOldRewindData(int height)
         {
-            lock (this.LockObject)
-            {
-                var itemsToPurge = this.addressIndexerRewindData.Find(x => x.BlockHeight < height).ToArray();
-
-                for (int i = 0; i < itemsToPurge.Count(); i++)
-                {
-                    this.addressIndexerRewindData.Delete(itemsToPurge[i].BlockHash);
+            // Delete all in one go based on query. This is more optimal than query, iterate and delete individual records.
+            int purgedCount = this.addressIndexerRewindData.Delete(x => x.BlockHeight < height);
 
-                    if (i % 100 == 0)
-                        this.logger.LogInformation("Purging {0}/{1} rewind data items.", i, itemsToPurge.Count());
-                }
-            }
+            this.logger.LogInformation("Purged {0} rewind data items.", purgedCount);
         }
 
         /// <summary>Reverts changes made by processing blocks with height higher than <param name="height">.</param></summary>
         /// <param name="height">The height above which to restore outpoints.</param>
         public void RewindDataAboveHeight(int height)
         {
-            lock (this.LockObject)
-            {
-                IEnumerable<AddressIndexerRewindData> toRestore = this.addressIndexerRewindData.Find(x => x.BlockHeight > height);
+            IEnumerable<AddressIndexerRewindData> toRestore = this.addressIndexerRewindData.Find(x => x.BlockHeight > height);
 
-                this.logger.LogDebug("Restoring data for {0} blocks.", toRestore.Count());
+            this.logger.LogDebug("Restoring data for {0} blocks.", toRestore.Count());
 
-                foreach (AddressIndexerRewindData rewindData in toRestore)
+            foreach (AddressIndexerRewindData rewindData in toRestore)
+            {
+                // Put the spent outputs back into the cache.
+                foreach (OutPointData outPointData in rewindData.SpentOutputs)
                 {
-                    // Put the spent outputs back into the cache.
-                    foreach (OutPointData outPointData in rewindData.SpentOutputs)
-                        this.AddOutPointData(outPointData);
-
-                    // This rewind data item should now be removed from the collection.
-                    this.addressIndexerRewindData.Delete(rewindData.BlockHash);
+                    this.AddOutPointData(outPointData);
                 }
+
+                // This rewind data item should now be removed from the collection.
+                this.addressIndexerRewindData.Delete(rewindData.BlockHash);
             }
         }