Chain Repository optimization

src/Blockcore.IntegrationTests/CoinViewTests.cs

@@ -279,10 +279,11 @@ private ChainedHeader MakeNext(ChainedHeader previous, Network network)
         [Fact]
         public void CanSaveChainIncrementally()
         {
-            using (var repo = new ChainRepository(TestBase.CreateTestDir(this), this.loggerFactory, this.dataStoreSerializer, new MemoryHeaderStore()))
-            {
-                var chain = new ChainIndexer(this.regTest);
+            var chain = new ChainIndexer(this.regTest);
+            var data = new DataFolder(TestBase.CreateTestDir(this));
 
+            using (var repo = new ChainRepository(this.loggerFactory, new LeveldbHeaderStore(this.network, data, chain), this.network))
+            {
                 chain.SetTip(repo.LoadAsync(chain.Genesis).GetAwaiter().GetResult());
                 Assert.True(chain.Tip == chain.Genesis);
                 chain = new ChainIndexer(this.regTest);

src/Blockcore.Tests/Base/ChainRepositoryTest.cs

@@ -1,12 +1,15 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using Blockcore.Base;
+using Blockcore.Configuration;
 using Blockcore.Tests.Common;
 using Blockcore.Utilities;
 using DBreeze;
 using DBreeze.DataTypes;
 using LevelDB;
 using Microsoft.Extensions.Logging;
+using Moq;
 using NBitcoin;
 using Xunit;
 
@@ -22,42 +25,45 @@ public ChainRepositoryTest() : base(KnownNetworks.StratisRegTest)
         }
 
         [Fact]
-        public void SaveWritesChainToDisk()
+        public void SaveChainToDisk()
         {
             string dir = CreateTestDir(this);
             var chain = new ChainIndexer(KnownNetworks.StratisRegTest);
             this.AppendBlock(chain);
 
-            using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dataStoreSerializer, new MemoryHeaderStore()))
+            using (var repo = new ChainRepository(new LoggerFactory(), new LeveldbHeaderStore(chain.Network, new DataFolder(dir), chain), chain.Network))
             {
                 repo.SaveAsync(chain).GetAwaiter().GetResult();
             }
 
-            using (var engine = new DB(new Options { CreateIfMissing = true }, dir))
+            using (var engine = new DB(new Options { CreateIfMissing = true }, new DataFolder(dir).ChainPath))
             {
                 ChainedHeader tip = null;
                 var itr = engine.GetEnumerator();
 
                 while (itr.MoveNext())
                 {
-                    var blockHeader = this.dataStoreSerializer.Deserialize<BlockHeader>(itr.Current.Value);
+                    if (itr.Current.Key[0] == 1)
+                    {
+                        var data = new ChainRepository.ChainRepositoryData();
+                        data.FromBytes(itr.Current.Value.ToArray(), this.Network.Consensus.ConsensusFactory);
 
-                    if (tip != null && blockHeader.HashPrevBlock != tip.HashBlock)
-                        break;
-                    tip = new ChainedHeader(blockHeader, blockHeader.GetHash(), tip);
+                        tip = new ChainedHeader(data.Hash, data.Work, tip);
+                        if (tip.Height == 0) tip.SetChainStore(new ChainStore());
+                    }
                 }
                 Assert.Equal(tip, chain.Tip);
             }
         }
 
         [Fact]
-        public void GetChainReturnsConcurrentChainFromDisk()
+        public void LoadChainFromDisk()
         {
             string dir = CreateTestDir(this);
             var chain = new ChainIndexer(KnownNetworks.StratisRegTest);
             ChainedHeader tip = this.AppendBlock(chain);
 
-            using (var engine = new DB(new Options { CreateIfMissing = true }, dir))
+            using (var engine = new DB(new Options { CreateIfMissing = true }, new DataFolder(dir).ChainPath))
             {
                 using (var batch = new WriteBatch())
                 {
@@ -71,13 +77,16 @@ public void GetChainReturnsConcurrentChainFromDisk()
 
                     foreach (ChainedHeader block in blocks)
                     {
-                        batch.Put(BitConverter.GetBytes(block.Height), this.dataStoreSerializer.Serialize(block.Header));
+                        batch.Put(DBH.Key(1, BitConverter.GetBytes(block.Height)),
+                            new ChainRepository.ChainRepositoryData()
+                            { Hash = block.HashBlock, Work = block.ChainWorkBytes }
+                                .ToBytes(this.Network.Consensus.ConsensusFactory));
                     }
 
                     engine.Write(batch);
                 }
             }
-            using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dataStoreSerializer, new MemoryHeaderStore()))
+            using (var repo = new ChainRepository(new LoggerFactory(), new LeveldbHeaderStore(chain.Network, new DataFolder(dir), chain), chain.Network))
             {
                 var testChain = new ChainIndexer(KnownNetworks.StratisRegTest);
                 testChain.SetTip(repo.LoadAsync(testChain.Genesis).GetAwaiter().GetResult());

src/Blockcore/Base/BaseFeature.cs

@@ -389,7 +389,7 @@ public static IFullNodeBuilder UseBaseFeature(this IFullNodeBuilder fullNodeBuil
                     services.AddSingleton<IInvalidBlockHashStore, InvalidBlockHashStore>();
                     services.AddSingleton<IChainState, ChainState>();
                     services.AddSingleton<IChainRepository, ChainRepository>();
-                    services.AddSingleton<IBlockHeaderStore, LeveldbHeaderStore>();
+                    services.AddSingleton<IChainStore, LeveldbHeaderStore>();
                     services.AddSingleton<IFinalizedBlockInfoRepository, FinalizedBlockInfoRepository>();
                     services.AddSingleton<ITimeSyncBehaviorState, TimeSyncBehaviorState>();
                     services.AddSingleton<NodeDeployments>();

src/Blockcore/Base/ChainRepository.cs

@@ -23,36 +23,23 @@ public interface IChainRepository : IDisposable
 
     public class ChainRepository : IChainRepository
     {
-        private readonly DataStoreSerializer dataStoreSerializer;
-        private readonly IBlockHeaderStore blockHeaderStore;
+        private readonly IChainStore chainStore;
 
         /// <summary>Instance logger.</summary>
         private readonly ILogger logger;
 
-        /// <summary>Access to database.</summary>
-        private readonly DB leveldb;
-
         private BlockLocator locator;
 
-        public ChainRepository(string folder, ILoggerFactory loggerFactory, DataStoreSerializer dataStoreSerializer, IBlockHeaderStore blockHeaderStore)
+        public Network Network { get; }
+
+        public ChainRepository(ILoggerFactory loggerFactory, IChainStore chainStore, Network network)
         {
-            this.dataStoreSerializer = dataStoreSerializer;
-            this.blockHeaderStore = blockHeaderStore;
-            Guard.NotEmpty(folder, nameof(folder));
             Guard.NotNull(loggerFactory, nameof(loggerFactory));
 
-            this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
-
-            Directory.CreateDirectory(folder);
+            this.chainStore = chainStore;
+            this.Network = network;
 
-            // Open a connection to a new DB and create if not found
-            var options = new Options { CreateIfMissing = true };
-            this.leveldb = new DB(options, folder);
-        }
-
-        public ChainRepository(DataFolder dataFolder, ILoggerFactory loggerFactory, DataStoreSerializer dataStoreSerializer, IBlockHeaderStore blockHeaderStore)
-            : this(dataFolder.ChainPath, loggerFactory, dataStoreSerializer, blockHeaderStore)
-        {
+            this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
         }
 
         /// <inheritdoc />
@@ -62,41 +49,32 @@ public Task<ChainedHeader> LoadAsync(ChainedHeader genesisHeader)
             {
                 ChainedHeader tip = null;
 
-                byte[] firstRow = this.leveldb.Get(BitConverter.GetBytes(0));
+                ChainData data = this.chainStore.GetChainData(0);
 
-                if (firstRow == null)
+                if (data == null)
                 {
-                    genesisHeader.SetBlockHeaderStore(this.blockHeaderStore);
+                    genesisHeader.SetChainStore(this.chainStore);
                     return genesisHeader;
                 }
 
-                BlockHeader nextHeader = this.dataStoreSerializer.Deserialize<BlockHeader>(firstRow);
-                Guard.Assert(nextHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks
+                Guard.Assert(data.Hash == genesisHeader.HashBlock); // can't swap networks
 
-                int index = 1;
+                int index = 0;
                 while (true)
                 {
-                    byte[] row = this.leveldb.Get(BitConverter.GetBytes(index));
-
-                    if (row == null)
-                        break;
+                    data = this.chainStore.GetChainData((index));
 
-                    if ((tip != null) && (nextHeader.HashPrevBlock != tip.HashBlock))
+                    if (data == null)
                         break;
 
-                    BlockHeader blockHeader = this.dataStoreSerializer.Deserialize<BlockHeader>(row);
-                    tip = new ChainedHeader(nextHeader, blockHeader.HashPrevBlock, tip);
-                    if (tip.Height == 0) tip.SetBlockHeaderStore(this.blockHeaderStore);
-                    nextHeader = blockHeader;
+                    tip = new ChainedHeader(data.Hash, data.Work, tip);
+                    if (tip.Height == 0) tip.SetChainStore(this.chainStore);
                     index++;
                 }
 
-                if (nextHeader != null)
-                    tip = new ChainedHeader(nextHeader, nextHeader.GetHash(), tip);
-
                 if (tip == null)
                 {
-                    genesisHeader.SetBlockHeaderStore(this.blockHeaderStore);
+                    genesisHeader.SetChainStore(this.chainStore);
                     tip = genesisHeader;
                 }
 
@@ -114,29 +92,26 @@ public Task SaveAsync(ChainIndexer chainIndexer)
 
             Task task = Task.Run(() =>
             {
-                using (var batch = new WriteBatch())
+                ChainedHeader fork = this.locator == null ? null : chainIndexer.FindFork(this.locator);
+                ChainedHeader tip = chainIndexer.Tip;
+                ChainedHeader toSave = tip;
+
+                var headers = new List<ChainedHeader>();
+                while (toSave != fork)
                 {
-                    ChainedHeader fork = this.locator == null ? null : chainIndexer.FindFork(this.locator);
-                    ChainedHeader tip = chainIndexer.Tip;
-                    ChainedHeader toSave = tip;
-
-                    var headers = new List<ChainedHeader>();
-                    while (toSave != fork)
-                    {
-                        headers.Add(toSave);
-                        toSave = toSave.Previous;
-                    }
-
-                    // DBreeze is faster on ordered insert.
-                    IOrderedEnumerable<ChainedHeader> orderedChainedHeaders = headers.OrderBy(b => b.Height);
-                    foreach (ChainedHeader block in orderedChainedHeaders)
-                    {
-                        batch.Put(BitConverter.GetBytes(block.Height), this.dataStoreSerializer.Serialize(block.Header));
-                    }
-
-                    this.locator = tip.GetLocator();
-                    this.leveldb.Write(batch);
+                    headers.Add(toSave);
+                    toSave = toSave.Previous;
                 }
+
+                var items = headers.OrderBy(b => b.Height).Select(h => new ChainDataItem
+                {
+                    Height = h.Height,
+                    Data = new ChainData { Hash = h.HashBlock, Work = h.ChainWorkBytes }
+                });
+
+                this.chainStore.PutChainData(items);
+
+                this.locator = tip.GetLocator();
             });
 
             return task;
@@ -145,8 +120,35 @@ public Task SaveAsync(ChainIndexer chainIndexer)
         /// <inheritdoc />
         public void Dispose()
         {
-            this.leveldb?.Dispose();
-            (this.blockHeaderStore as IDisposable)?.Dispose();
+            (this.chainStore as IDisposable)?.Dispose();
+        }
+
+        public class ChainRepositoryData : IBitcoinSerializable
+        {
+            public uint256 Hash;
+            public byte[] Work;
+
+            public ChainRepositoryData()
+            {
+            }
+
+            public void ReadWrite(BitcoinStream stream)
+            {
+                stream.ReadWrite(ref this.Hash);
+                if (stream.Serializing)
+                {
+                    int len = this.Work.Length;
+                    stream.ReadWrite(ref len);
+                    stream.ReadWrite(ref this.Work);
+                }
+                else
+                {
+                    int len = 0;
+                    stream.ReadWrite(ref len);
+                    this.Work = new byte[len];
+                    stream.ReadWrite(ref this.Work);
+                }
+            }
         }
     }
 }

src/Blockcore/Configuration/DataFolder.cs

@@ -25,7 +25,6 @@ public DataFolder(string path)
             this.CoindbPath = Path.Combine(path, "coindb");
             this.AddressManagerFilePath = path;
             this.ChainPath = Path.Combine(path, "chain");
-            this.HeadersPath = Path.Combine(path, "headers");
             this.KeyValueRepositoryPath = Path.Combine(path, "common");
             this.BlockPath = Path.Combine(path, "blocks");
             this.PollsPath = Path.Combine(path, "polls");
@@ -56,8 +55,6 @@ public DataFolder(string path)
         /// <seealso cref="Base.BaseFeature.StartChain"/>
         public string ChainPath { get; internal set; }
 
-        public string HeadersPath { get; internal set; }
-
         /// <summary>Path to the folder with separated key-value items managed by <see cref="IKeyValueRepository"/>.</summary>
         public string KeyValueRepositoryPath { get; internal set; }