Move to leveldb block store, provenheader and chainrepo (#90)

* Change store, provenheader and chainrepo

* Add lock to proven header

* Fix some more tests

* Limit the number of unconsumed block count in consensus manager

* Fix tests

* Move finalized repo to consensus ns

Author: dangershony

src/Blockcore.Features.BlockStore.Tests/BlockRepositoryTests.cs

@@ -3,6 +3,7 @@
 using System.Threading.Tasks;
 using Blockcore.Utilities;
 using DBreeze.DataTypes;
+using LevelDB;
 using Microsoft.Extensions.Logging;
 using NBitcoin;
 
@@ -31,10 +32,7 @@ public PrunedBlockRepository(IBlockRepository blockRepository, DBreezeSerializer
         /// <inheritdoc />
         public void Initialize()
         {
-            using (DBreeze.Transactions.Transaction transaction = this.blockRepository.DBreeze.GetTransaction())
-            {
-                this.LoadPrunedTip(transaction);
-            }
+            this.LoadPrunedTip(this.blockRepository.Leveldb);
         }
 
         /// <inheritdoc />
@@ -48,11 +46,7 @@ public void PruneAndCompactDatabase(ChainedHeader blockRepositoryTip, Network ne
 
                 this.PrunedTip = new HashHeightPair(genesis.GetHash(), 0);
 
-                using (DBreeze.Transactions.Transaction transaction = this.blockRepository.DBreeze.GetTransaction())
-                {
-                    transaction.Insert(BlockRepository.CommonTableName, prunedTipKey, this.dBreezeSerializer.Serialize(this.PrunedTip));
-                    transaction.Commit();
-                }
+                this.blockRepository.Leveldb.Put(DBH.Key(BlockRepository.CommonTableName, prunedTipKey), this.dBreezeSerializer.Serialize(this.PrunedTip));
             }
 
             if (nodeInitializing)
@@ -110,17 +104,13 @@ private void PrepareDatabaseForCompacting(ChainedHeader blockRepositoryTip)
             this.UpdatePrunedTip(blockRepositoryTip.GetAncestor(upperHeight));
         }
 
-        private void LoadPrunedTip(DBreeze.Transactions.Transaction dbreezeTransaction)
+        private void LoadPrunedTip(DB leveldb)
         {
             if (this.PrunedTip == null)
             {
-                dbreezeTransaction.ValuesLazyLoadingIsOn = false;
-
-                Row<byte[], byte[]> row = dbreezeTransaction.Select<byte[], byte[]>(BlockRepository.CommonTableName, prunedTipKey);
-                if (row.Exists)
-                    this.PrunedTip = this.dBreezeSerializer.Deserialize<HashHeightPair>(row.Value);
-
-                dbreezeTransaction.ValuesLazyLoadingIsOn = true;
+                byte[] row = leveldb.Get(DBH.Key(BlockRepository.CommonTableName, prunedTipKey));
+                if (row != null)
+                    this.PrunedTip = this.dBreezeSerializer.Deserialize<HashHeightPair>(row);
             }
         }
 
@@ -131,34 +121,6 @@ private void CompactDataBase()
         {
             Task task = Task.Run(() =>
             {
-                using (DBreeze.Transactions.Transaction dbreezeTransaction = this.blockRepository.DBreeze.GetTransaction())
-                {
-                    dbreezeTransaction.SynchronizeTables(BlockRepository.BlockTableName, BlockRepository.TransactionTableName);
-
-                    var tempBlocks = dbreezeTransaction.SelectDictionary<byte[], byte[]>(BlockRepository.BlockTableName);
-
-                    if (tempBlocks.Count != 0)
-                    {
-                        this.logger.LogInformation($"{tempBlocks.Count} blocks will be copied to the pruned table.");
-
-                        dbreezeTransaction.RemoveAllKeys(BlockRepository.BlockTableName, true);
-                        dbreezeTransaction.InsertDictionary(BlockRepository.BlockTableName, tempBlocks, false);
-
-                        var tempTransactions = dbreezeTransaction.SelectDictionary<byte[], byte[]>(BlockRepository.TransactionTableName);
-                        if (tempTransactions.Count != 0)
-                        {
-                            this.logger.LogInformation($"{tempTransactions.Count} transactions will be copied to the pruned table.");
-                            dbreezeTransaction.RemoveAllKeys(BlockRepository.TransactionTableName, true);
-                            dbreezeTransaction.InsertDictionary(BlockRepository.TransactionTableName, tempTransactions, false);
-                        }
-
-                        // Save the hash and height of where the node was pruned up to.
-                        dbreezeTransaction.Insert(BlockRepository.CommonTableName, prunedTipKey, this.dBreezeSerializer.Serialize(this.PrunedTip));
-                    }
-
-                    dbreezeTransaction.Commit();
-                }
-
                 return Task.CompletedTask;
             });
         }
@@ -169,4 +131,4 @@ public void UpdatePrunedTip(ChainedHeader tip)
             this.PrunedTip = new HashHeightPair(tip);
         }
     }
-}
+}

src/Blockcore.Features.BlockStore/BlockRepository.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
 using Blockcore.Features.Consensus.ProvenBlockHeaders;
@@ -9,6 +10,7 @@
 using DBreeze;
 using DBreeze.Utils;
 using FluentAssertions;
+using LevelDB;
 using Microsoft.Extensions.Logging;
 using Moq;
 using NBitcoin;
@@ -20,8 +22,8 @@ public class ProvenBlockHeaderRepositoryTests : LogsTestBase
     {
         private readonly Mock<ILoggerFactory> loggerFactory;
         private readonly DBreezeSerializer dBreezeSerializer;
-        private const string ProvenBlockHeaderTable = "ProvenBlockHeader";
-        private const string BlockHashTable = "BlockHashHeight";
+        private static readonly byte ProvenBlockHeaderTable = 1;
+        private static readonly byte BlockHashHeightTable = 2;
 
         public ProvenBlockHeaderRepositoryTests() : base(KnownNetworks.StratisTest)
         {
@@ -58,14 +60,10 @@ public async Task PutAsync_WritesProvenBlockHeaderAndSavesBlockHashAsync()
                 await repo.PutAsync(items, blockHashHeightPair);
             }
 
-            using (var engine = new DBreezeEngine(folder))
+            using (var engine = new DB(new Options() { CreateIfMissing = true }, folder))
             {
-                DBreeze.Transactions.Transaction txn = engine.GetTransaction();
-                txn.SynchronizeTables(ProvenBlockHeaderTable);
-                txn.ValuesLazyLoadingIsOn = false;
-
-                var headerOut = this.dBreezeSerializer.Deserialize<ProvenBlockHeader>(txn.Select<byte[], byte[]>(ProvenBlockHeaderTable, blockHashHeightPair.Height.ToBytes()).Value);
-                var hashHeightPairOut = this.DBreezeSerializer.Deserialize<HashHeightPair>(txn.Select<byte[], byte[]>(BlockHashTable, new byte[0].ToBytes()).Value);
+                var headerOut = this.dBreezeSerializer.Deserialize<ProvenBlockHeader>(engine.Get(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHashHeightPair.Height))));
+                var hashHeightPairOut = this.DBreezeSerializer.Deserialize<HashHeightPair>(engine.Get(DBH.Key(BlockHashHeightTable, new byte[] { 1 })));
 
                 headerOut.Should().NotBeNull();
                 headerOut.GetHash().Should().Be(provenBlockHeaderIn.GetHash());
@@ -93,13 +91,13 @@ public async Task PutAsync_Inserts_MultipleProvenBlockHeadersAsync()
             }
 
             // Check the ProvenBlockHeader exists in the database.
-            using (var engine = new DBreezeEngine(folder))
+            using (var engine = new DB(new Options() { CreateIfMissing = true }, folder))
             {
-                DBreeze.Transactions.Transaction txn = engine.GetTransaction();
-                txn.SynchronizeTables(ProvenBlockHeaderTable);
-                txn.ValuesLazyLoadingIsOn = false;
-
-                var headersOut = txn.SelectDictionary<byte[], byte[]>(ProvenBlockHeaderTable);
+                var headersOut = new Dictionary<byte[], byte[]>();
+                var enumeator = engine.GetEnumerator();
+                while (enumeator.MoveNext())
+                    if (enumeator.Current.Key[0] == ProvenBlockHeaderTable)
+                        headersOut.Add(enumeator.Current.Key, enumeator.Current.Value);
 
                 headersOut.Keys.Count.Should().Be(2);
                 this.dBreezeSerializer.Deserialize<ProvenBlockHeader>(headersOut.First().Value).GetHash().Should().Be(items[0].GetHash());
@@ -116,11 +114,9 @@ public async Task GetAsync_ReadsProvenBlockHeaderAsync()
 
             int blockHeight = 1;
 
-            using (var engine = new DBreezeEngine(folder))
+            using (var engine = new DB(new Options() { CreateIfMissing = true }, folder))
             {
-                DBreeze.Transactions.Transaction txn = engine.GetTransaction();
-                txn.Insert<byte[], byte[]>(ProvenBlockHeaderTable, blockHeight.ToBytes(), this.dBreezeSerializer.Serialize(headerIn));
-                txn.Commit();
+                engine.Put(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight)), this.dBreezeSerializer.Serialize(headerIn));
             }
 
             // Query the repository for the item that was inserted in the above code.
@@ -138,12 +134,10 @@ public async Task GetAsync_WithWrongBlockHeightReturnsNullAsync()
         {
             string folder = CreateTestDir(this);
 
-            using (var engine = new DBreezeEngine(folder))
+            using (var engine = new DB(new Options() { CreateIfMissing = true }, folder))
             {
-                DBreeze.Transactions.Transaction txn = engine.GetTransaction();
-                txn.Insert<byte[], byte[]>(ProvenBlockHeaderTable, 1.ToBytes(), this.dBreezeSerializer.Serialize(CreateNewProvenBlockHeaderMock()));
-                txn.Insert<byte[], byte[]>(BlockHashTable, new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(), 1)));
-                txn.Commit();
+                engine.Put(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(1)), this.dBreezeSerializer.Serialize(CreateNewProvenBlockHeaderMock()));
+                engine.Put(DBH.Key(BlockHashHeightTable, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(), 1)));
             }
 
             using (ProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder))
@@ -159,7 +153,7 @@ public async Task GetAsync_WithWrongBlockHeightReturnsNullAsync()
         }
 
         [Fact]
-        public async Task PutAsync_Add_Ten_ProvenBlockHeaders_Dispose_On_Initialise_Repo_TipHeight_Should_Be_At_Last_Saved_TipAsync()
+        public async Task PutAsync_DisposeOnInitialise_ShouldBeAtLastSavedTipAsync()
         {
             string folder = CreateTestDir(this);
 
@@ -195,4 +189,4 @@ private ProvenBlockHeaderRepository SetupRepository(Network network, string fold
             return repo;
         }
     }
-}
+}