diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip index 32d9b3c8a..a60a885da 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip index 2f3daaffd..d5a3cbc39 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip index 986af81f6..094760a7b 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip index 699f659cd..2db747c35 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip index 7801f0dd8..f3d14bb85 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip index e4c7193d5..5b1b85013 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip index 2cc99d200..80f55791f 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip index 02960ce4a..5f7a31889 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip index 14b38dd5b..1e41c493e 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip index 2c44778e6..a2e6d5a03 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip index 7ae1f1338..13746c86f 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip index e49560bea..851c1e291 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip index 5d5163f0d..cfbcccf5d 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip index bdd67dfff..6a5c823c0 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip index 19e69e75c..6538288d7 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip index 739550dec..4193615cf 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip index 5f19b4bc9..d76bd1daf 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip index ce1ef1321..6789e3ed1 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests/CoinViewTests.cs b/src/Blockcore.IntegrationTests/CoinViewTests.cs index f059d2f50..a4e7e6c2f 100644 --- a/src/Blockcore.IntegrationTests/CoinViewTests.cs +++ b/src/Blockcore.IntegrationTests/CoinViewTests.cs @@ -279,10 +279,11 @@ private ChainedHeader MakeNext(ChainedHeader previous, Network network) [Fact] public void CanSaveChainIncrementally() { - using (var repo = new ChainRepository(TestBase.CreateTestDir(this), this.loggerFactory, this.dataStoreSerializer, new MemoryHeaderStore())) - { - var chain = new ChainIndexer(this.regTest); + var chain = new ChainIndexer(this.regTest); + var data = new DataFolder(TestBase.CreateTestDir(this)); + using (var repo = new ChainRepository(this.loggerFactory, new LeveldbHeaderStore(this.network, data, chain), this.network)) + { chain.SetTip(repo.LoadAsync(chain.Genesis).GetAwaiter().GetResult()); Assert.True(chain.Tip == chain.Genesis); chain = new ChainIndexer(this.regTest); diff --git a/src/Blockcore.Tests/Base/ChainRepositoryTest.cs b/src/Blockcore.Tests/Base/ChainRepositoryTest.cs index 8d6913b01..5a50a65fe 100644 --- a/src/Blockcore.Tests/Base/ChainRepositoryTest.cs +++ b/src/Blockcore.Tests/Base/ChainRepositoryTest.cs @@ -1,12 +1,15 @@ using System; using System.Collections.Generic; +using System.Linq; using Blockcore.Base; +using Blockcore.Configuration; using Blockcore.Tests.Common; using Blockcore.Utilities; using DBreeze; using DBreeze.DataTypes; using LevelDB; using Microsoft.Extensions.Logging; +using Moq; using NBitcoin; using Xunit; @@ -22,42 +25,45 @@ public ChainRepositoryTest() : base(KnownNetworks.StratisRegTest) } [Fact] - public void SaveWritesChainToDisk() + public void SaveChainToDisk() { string dir = CreateTestDir(this); var chain = new ChainIndexer(KnownNetworks.StratisRegTest); this.AppendBlock(chain); - using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dataStoreSerializer, new MemoryHeaderStore())) + using (var repo = new ChainRepository(new LoggerFactory(), new LeveldbHeaderStore(chain.Network, new DataFolder(dir), chain), chain.Network)) { repo.SaveAsync(chain).GetAwaiter().GetResult(); } - using (var engine = new DB(new Options { CreateIfMissing = true }, dir)) + using (var engine = new DB(new Options { CreateIfMissing = true }, new DataFolder(dir).ChainPath)) { ChainedHeader tip = null; var itr = engine.GetEnumerator(); while (itr.MoveNext()) { - var blockHeader = this.dataStoreSerializer.Deserialize(itr.Current.Value); + if (itr.Current.Key[0] == 1) + { + var data = new ChainRepository.ChainRepositoryData(); + data.FromBytes(itr.Current.Value.ToArray(), this.Network.Consensus.ConsensusFactory); - if (tip != null && blockHeader.HashPrevBlock != tip.HashBlock) - break; - tip = new ChainedHeader(blockHeader, blockHeader.GetHash(), tip); + tip = new ChainedHeader(data.Hash, data.Work, tip); + if (tip.Height == 0) tip.SetChainStore(new ChainStore()); + } } Assert.Equal(tip, chain.Tip); } } [Fact] - public void GetChainReturnsConcurrentChainFromDisk() + public void LoadChainFromDisk() { string dir = CreateTestDir(this); var chain = new ChainIndexer(KnownNetworks.StratisRegTest); ChainedHeader tip = this.AppendBlock(chain); - using (var engine = new DB(new Options { CreateIfMissing = true }, dir)) + using (var engine = new DB(new Options { CreateIfMissing = true }, new DataFolder(dir).ChainPath)) { using (var batch = new WriteBatch()) { @@ -71,13 +77,16 @@ public void GetChainReturnsConcurrentChainFromDisk() foreach (ChainedHeader block in blocks) { - batch.Put(BitConverter.GetBytes(block.Height), this.dataStoreSerializer.Serialize(block.Header)); + batch.Put(DBH.Key(1, BitConverter.GetBytes(block.Height)), + new ChainRepository.ChainRepositoryData() + { Hash = block.HashBlock, Work = block.ChainWorkBytes } + .ToBytes(this.Network.Consensus.ConsensusFactory)); } engine.Write(batch); } } - using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dataStoreSerializer, new MemoryHeaderStore())) + using (var repo = new ChainRepository(new LoggerFactory(), new LeveldbHeaderStore(chain.Network, new DataFolder(dir), chain), chain.Network)) { var testChain = new ChainIndexer(KnownNetworks.StratisRegTest); testChain.SetTip(repo.LoadAsync(testChain.Genesis).GetAwaiter().GetResult()); diff --git a/src/Blockcore/Base/BaseFeature.cs b/src/Blockcore/Base/BaseFeature.cs index aa05cec94..568373df1 100644 --- a/src/Blockcore/Base/BaseFeature.cs +++ b/src/Blockcore/Base/BaseFeature.cs @@ -389,7 +389,7 @@ public static IFullNodeBuilder UseBaseFeature(this IFullNodeBuilder fullNodeBuil services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/Blockcore/Base/ChainRepository.cs b/src/Blockcore/Base/ChainRepository.cs index 1b3a3b35f..4cc5a0238 100644 --- a/src/Blockcore/Base/ChainRepository.cs +++ b/src/Blockcore/Base/ChainRepository.cs @@ -23,36 +23,23 @@ public interface IChainRepository : IDisposable public class ChainRepository : IChainRepository { - private readonly DataStoreSerializer dataStoreSerializer; - private readonly IBlockHeaderStore blockHeaderStore; + private readonly IChainStore chainStore; /// Instance logger. private readonly ILogger logger; - /// Access to database. - private readonly DB leveldb; - private BlockLocator locator; - public ChainRepository(string folder, ILoggerFactory loggerFactory, DataStoreSerializer dataStoreSerializer, IBlockHeaderStore blockHeaderStore) + public Network Network { get; } + + public ChainRepository(ILoggerFactory loggerFactory, IChainStore chainStore, Network network) { - this.dataStoreSerializer = dataStoreSerializer; - this.blockHeaderStore = blockHeaderStore; - Guard.NotEmpty(folder, nameof(folder)); Guard.NotNull(loggerFactory, nameof(loggerFactory)); - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - - Directory.CreateDirectory(folder); + this.chainStore = chainStore; + this.Network = network; - // Open a connection to a new DB and create if not found - var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, folder); - } - - public ChainRepository(DataFolder dataFolder, ILoggerFactory loggerFactory, DataStoreSerializer dataStoreSerializer, IBlockHeaderStore blockHeaderStore) - : this(dataFolder.ChainPath, loggerFactory, dataStoreSerializer, blockHeaderStore) - { + this.logger = loggerFactory.CreateLogger(this.GetType().FullName); } /// @@ -62,41 +49,32 @@ public Task LoadAsync(ChainedHeader genesisHeader) { ChainedHeader tip = null; - byte[] firstRow = this.leveldb.Get(BitConverter.GetBytes(0)); + ChainData data = this.chainStore.GetChainData(0); - if (firstRow == null) + if (data == null) { - genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); + genesisHeader.SetChainStore(this.chainStore); return genesisHeader; } - BlockHeader nextHeader = this.dataStoreSerializer.Deserialize(firstRow); - Guard.Assert(nextHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks + Guard.Assert(data.Hash == genesisHeader.HashBlock); // can't swap networks - int index = 1; + int index = 0; while (true) { - byte[] row = this.leveldb.Get(BitConverter.GetBytes(index)); - - if (row == null) - break; + data = this.chainStore.GetChainData((index)); - if ((tip != null) && (nextHeader.HashPrevBlock != tip.HashBlock)) + if (data == null) break; - BlockHeader blockHeader = this.dataStoreSerializer.Deserialize(row); - tip = new ChainedHeader(nextHeader, blockHeader.HashPrevBlock, tip); - if (tip.Height == 0) tip.SetBlockHeaderStore(this.blockHeaderStore); - nextHeader = blockHeader; + tip = new ChainedHeader(data.Hash, data.Work, tip); + if (tip.Height == 0) tip.SetChainStore(this.chainStore); index++; } - if (nextHeader != null) - tip = new ChainedHeader(nextHeader, nextHeader.GetHash(), tip); - if (tip == null) { - genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); + genesisHeader.SetChainStore(this.chainStore); tip = genesisHeader; } @@ -114,29 +92,26 @@ public Task SaveAsync(ChainIndexer chainIndexer) Task task = Task.Run(() => { - using (var batch = new WriteBatch()) + ChainedHeader fork = this.locator == null ? null : chainIndexer.FindFork(this.locator); + ChainedHeader tip = chainIndexer.Tip; + ChainedHeader toSave = tip; + + var headers = new List(); + while (toSave != fork) { - ChainedHeader fork = this.locator == null ? null : chainIndexer.FindFork(this.locator); - ChainedHeader tip = chainIndexer.Tip; - ChainedHeader toSave = tip; - - var headers = new List(); - while (toSave != fork) - { - headers.Add(toSave); - toSave = toSave.Previous; - } - - // DBreeze is faster on ordered insert. - IOrderedEnumerable orderedChainedHeaders = headers.OrderBy(b => b.Height); - foreach (ChainedHeader block in orderedChainedHeaders) - { - batch.Put(BitConverter.GetBytes(block.Height), this.dataStoreSerializer.Serialize(block.Header)); - } - - this.locator = tip.GetLocator(); - this.leveldb.Write(batch); + headers.Add(toSave); + toSave = toSave.Previous; } + + var items = headers.OrderBy(b => b.Height).Select(h => new ChainDataItem + { + Height = h.Height, + Data = new ChainData { Hash = h.HashBlock, Work = h.ChainWorkBytes } + }); + + this.chainStore.PutChainData(items); + + this.locator = tip.GetLocator(); }); return task; @@ -145,8 +120,35 @@ public Task SaveAsync(ChainIndexer chainIndexer) /// public void Dispose() { - this.leveldb?.Dispose(); - (this.blockHeaderStore as IDisposable)?.Dispose(); + (this.chainStore as IDisposable)?.Dispose(); + } + + public class ChainRepositoryData : IBitcoinSerializable + { + public uint256 Hash; + public byte[] Work; + + public ChainRepositoryData() + { + } + + public void ReadWrite(BitcoinStream stream) + { + stream.ReadWrite(ref this.Hash); + if (stream.Serializing) + { + int len = this.Work.Length; + stream.ReadWrite(ref len); + stream.ReadWrite(ref this.Work); + } + else + { + int len = 0; + stream.ReadWrite(ref len); + this.Work = new byte[len]; + stream.ReadWrite(ref this.Work); + } + } } } } \ No newline at end of file diff --git a/src/Blockcore/Configuration/DataFolder.cs b/src/Blockcore/Configuration/DataFolder.cs index d2d273b39..f8c83374b 100644 --- a/src/Blockcore/Configuration/DataFolder.cs +++ b/src/Blockcore/Configuration/DataFolder.cs @@ -25,7 +25,6 @@ public DataFolder(string path) this.CoindbPath = Path.Combine(path, "coindb"); this.AddressManagerFilePath = path; this.ChainPath = Path.Combine(path, "chain"); - this.HeadersPath = Path.Combine(path, "headers"); this.KeyValueRepositoryPath = Path.Combine(path, "common"); this.BlockPath = Path.Combine(path, "blocks"); this.PollsPath = Path.Combine(path, "polls"); @@ -56,8 +55,6 @@ public DataFolder(string path) /// public string ChainPath { get; internal set; } - public string HeadersPath { get; internal set; } - /// Path to the folder with separated key-value items managed by . public string KeyValueRepositoryPath { get; internal set; } diff --git a/src/Blockcore/Consensus/LeveldbHeaderStore.cs b/src/Blockcore/Consensus/LeveldbHeaderStore.cs index 75e15453d..fff78c6c5 100644 --- a/src/Blockcore/Consensus/LeveldbHeaderStore.cs +++ b/src/Blockcore/Consensus/LeveldbHeaderStore.cs @@ -1,14 +1,18 @@ using System; +using System.Collections.Generic; using Blockcore.Configuration; using Blockcore.Utilities; using LevelDB; namespace NBitcoin { - public class LeveldbHeaderStore : IBlockHeaderStore, IDisposable + public class LeveldbHeaderStore : IChainStore, IDisposable { private readonly Network network; + internal static readonly byte ChainTableName = 1; + internal static readonly byte HeaderTableName = 2; + /// /// Headers that are close to the tip /// @@ -28,7 +32,7 @@ public LeveldbHeaderStore(Network network, DataFolder dataFolder, ChainIndexer c // Open a connection to a new DB and create if not found var options = new Options { CreateIfMissing = true }; - this.leveldb = new DB(options, dataFolder.HeadersPath); + this.leveldb = new DB(options, dataFolder.ChainPath); } public ChainIndexer ChainIndexer { get; } @@ -40,11 +44,11 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) return blockHeader; } - byte[] bytes = hash.ToBytes(); + ReadOnlySpan bytes = hash.ToReadOnlySpan(); lock (this.locker) { - bytes = this.leveldb.Get(bytes); + bytes = this.leveldb.Get(DBH.Key(HeaderTableName, bytes)); } if (bytes == null) @@ -53,7 +57,7 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) } blockHeader = this.network.Consensus.ConsensusFactory.CreateBlockHeader(); - blockHeader.FromBytes(bytes, this.network.Consensus.ConsensusFactory); + blockHeader.FromBytes(bytes.ToArray(), this.network.Consensus.ConsensusFactory); // If the header is 500 blocks behind tip or 100 blocks ahead cache it. if ((chainedHeader.Height > this.ChainIndexer.Height - 500) && (chainedHeader.Height <= this.ChainIndexer.Height + 100)) @@ -62,18 +66,54 @@ public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) return blockHeader; } - public bool StoreHeader(BlockHeader blockHeader) + public bool PutHeader(BlockHeader blockHeader) { ConsensusFactory consensusFactory = this.network.Consensus.ConsensusFactory; lock (this.locker) { - this.leveldb.Put(blockHeader.GetHash().ToBytes(), blockHeader.ToBytes(consensusFactory)); + this.leveldb.Put(DBH.Key(HeaderTableName, blockHeader.GetHash().ToReadOnlySpan()), blockHeader.ToBytes(consensusFactory)); } return true; } + public ChainData GetChainData(int height) + { + byte[] bytes = null; + + lock (this.locker) + { + bytes = this.leveldb.Get(DBH.Key(ChainTableName, BitConverter.GetBytes(height))); + } + + if (bytes == null) + { + return null; + } + + var data = new ChainData(); + data.FromBytes(bytes, this.network.Consensus.ConsensusFactory); + + return data; + } + + public void PutChainData(IEnumerable items) + { + using (var batch = new WriteBatch()) + { + foreach (var item in items) + { + batch.Put(DBH.Key(ChainTableName, BitConverter.GetBytes(item.Height)), item.Data.ToBytes(this.network.Consensus.ConsensusFactory)); + } + + lock (this.locker) + { + this.leveldb.Write(batch); + } + } + } + public void Dispose() { this.leveldb?.Dispose(); diff --git a/src/Blockcore/Utilities/LeveldbHelper.cs b/src/Blockcore/Utilities/LeveldbHelper.cs index cbfecdeec..9ea12486b 100644 --- a/src/Blockcore/Utilities/LeveldbHelper.cs +++ b/src/Blockcore/Utilities/LeveldbHelper.cs @@ -14,6 +14,14 @@ public static byte[] Key(byte table, byte[] key) return dbkey.ToArray(); } + public static byte[] Key(byte table, ReadOnlySpan key) + { + Span dbkey = stackalloc byte[key.Length + 1]; + dbkey[0] = table; + key.CopyTo(dbkey.Slice(1)); + return dbkey.ToArray(); + } + public static Dictionary SelectDictionary(this DB db, byte table) { var dict = new Dictionary(); diff --git a/src/Blockcore/SerializableResult.cs b/src/Blockcore/Utilities/SerializableResult.cs similarity index 97% rename from src/Blockcore/SerializableResult.cs rename to src/Blockcore/Utilities/SerializableResult.cs index 1f196076b..2d6b0920f 100644 --- a/src/Blockcore/SerializableResult.cs +++ b/src/Blockcore/Utilities/SerializableResult.cs @@ -1,6 +1,6 @@ using Newtonsoft.Json; -namespace Blockcore +namespace Blockcore.Utilities { /// /// A generic result type that can be serialized. diff --git a/src/NBitcoin/ChainStore.cs b/src/NBitcoin/ChainStore.cs new file mode 100644 index 000000000..2e9659f70 --- /dev/null +++ b/src/NBitcoin/ChainStore.cs @@ -0,0 +1,96 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using NBitcoin.BouncyCastle.Math; +using NBitcoin.Crypto; + +namespace NBitcoin +{ + public interface IChainStore + { + BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash); + + bool PutHeader(BlockHeader blockHeader); + + ChainData GetChainData(int height); + + void PutChainData(IEnumerable items); + } + + public class ChainData : IBitcoinSerializable + { + public uint256 Hash; + public byte[] Work; + + public ChainData() + { + } + + public void ReadWrite(BitcoinStream stream) + { + stream.ReadWrite(ref this.Hash); + if (stream.Serializing) + { + int len = this.Work.Length; + stream.ReadWrite(ref len); + stream.ReadWrite(ref this.Work); + } + else + { + int len = 0; + stream.ReadWrite(ref len); + this.Work = new byte[len]; + stream.ReadWrite(ref this.Work); + } + } + } + + public class ChainDataItem + { + public int Height { get; set; } + public ChainData Data { get; set; } + } + + public class ChainStore : IChainStore + { + private readonly ConcurrentDictionary headers; + private readonly ConcurrentDictionary chainData; + + public ChainStore() + { + this.headers = new ConcurrentDictionary(); + this.chainData = new ConcurrentDictionary(); + } + + public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) + { + if (!this.headers.TryGetValue(hash, out BlockHeader header)) + { + throw new ApplicationException("Header must exist if requested"); + } + + return header; + } + + public bool PutHeader(BlockHeader blockHeader) + { + return this.headers.TryAdd(blockHeader.GetHash(), blockHeader); + } + + public ChainData GetChainData(int height) + { + if (!this.chainData.TryGetValue(height, out ChainData data)) + { + throw new ApplicationException("ChainData must exist if requested"); + } + + return data; + } + + public void PutChainData(IEnumerable items) + { + foreach (ChainDataItem item in items) + this.chainData.TryAdd(item.Height, item.Data); + } + } +} \ No newline at end of file diff --git a/src/NBitcoin/ChainedHeader.cs b/src/NBitcoin/ChainedHeader.cs index 09ec12e47..b75d2e150 100644 --- a/src/NBitcoin/ChainedHeader.cs +++ b/src/NBitcoin/ChainedHeader.cs @@ -81,7 +81,7 @@ public BlockHeader Header { get { - return this.HeaderStore.GetHeader(this, this.HashBlock); + return this.ChainStore.GetHeader(this, this.HashBlock); } } @@ -94,16 +94,14 @@ public BlockHeader Header /// public ProvenBlockHeader ProvenBlockHeader { get; set; } - /// Integer representation of the . - /// The chain work field is represented as a byte array to reduce the memory foot print of a BigInteger - private byte[] chainWork; + public byte[] ChainWorkBytes { get; private set; } /// Total amount of work in the chain up to and including this block. public uint256 ChainWork { get { - return Target.ToUInt256(this.chainWork); + return Target.ToUInt256(this.ChainWorkBytes); } } @@ -113,7 +111,7 @@ public uint256 ChainWork /// public ValidationState BlockValidationState { get; set; } - public IBlockHeaderStore HeaderStore { get; private set; } + public IChainStore ChainStore { get; private set; } /// /// An indicator that the current instance of has been disconnected from the previous instance. @@ -138,17 +136,53 @@ public bool IsReferenceConnected public List Next { get; private set; } /// - /// Set a different header store to the default , this can be done only on genesis header (height 0). + /// Set a different header store to the default , this can be done only on genesis header (height 0). /// - public void SetBlockHeaderStore(IBlockHeaderStore blockHeaderStore) + public void SetChainStore(IChainStore chainStore) { if (this.Height != 0) { throw new ArgumentException("IBlockHeaderStore can only be set on the genesis header."); } - blockHeaderStore.StoreHeader(this.HeaderStore.GetHeader(this, this.HashBlock)); - this.HeaderStore = blockHeaderStore; + if (this.ChainStore != null) + chainStore.PutHeader(this.ChainStore.GetHeader(this, this.HashBlock)); + this.ChainStore = chainStore; + } + + public ChainedHeader(uint256 headerHash, byte[] chainWork, ChainedHeader previous) + { + this.HashBlock = headerHash ?? throw new ArgumentNullException(nameof(headerHash)); + this.Next = new List(1); + + if (previous != null) + this.Height = previous.Height + 1; + + this.Previous = previous; + + if (previous == null) + { + if (this.Height != 0) + throw new ArgumentException("Only the genesis block can have no previous block"); + } + else + { + // Calculates the location of the skip block for this block. + this.Skip = this.Previous.GetAncestor(this.GetSkipHeight(this.Height)); + + if (this.Previous.ChainStore == null) + throw new ArgumentException("ChainedHeader.Previous.HeaderStore was not found"); + + this.ChainStore = this.Previous.ChainStore; + } + + if (this.Height == 0) + { + this.BlockDataAvailability = BlockDataAvailabilityState.BlockAvailable; + this.BlockValidationState = ValidationState.FullyValidated; + } + + this.ChainWorkBytes = chainWork; } public ChainedHeader(ProvenBlockHeader header, uint256 headerHash, ChainedHeader previous) : this(header.PosBlockHeader, headerHash, previous) @@ -182,13 +216,13 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, ChainedHeader previ this.BlockDataAvailability = BlockDataAvailabilityState.BlockAvailable; this.BlockValidationState = ValidationState.FullyValidated; - this.HeaderStore = new MemoryHeaderStore(); - this.HeaderStore.StoreHeader(header); + this.ChainStore = new ChainStore(); + this.ChainStore.PutHeader(header); } else { - this.HeaderStore = this.Previous.HeaderStore; - this.HeaderStore.StoreHeader(header); + this.ChainStore = this.Previous.ChainStore; + this.ChainStore.PutHeader(header); } this.CalculateChainWork(); @@ -208,8 +242,8 @@ public ChainedHeader(BlockHeader header, uint256 headerHash, int height) : this( this.BlockValidationState = ValidationState.FullyValidated; } - this.HeaderStore = this.Previous?.HeaderStore ?? new MemoryHeaderStore(); - this.HeaderStore.StoreHeader(header); + this.ChainStore = this.Previous?.ChainStore ?? new ChainStore(); + this.ChainStore.PutHeader(header); this.CalculateChainWork(); @@ -235,8 +269,8 @@ private ChainedHeader(BlockHeader header, uint256 headerHash) /// private void CalculateChainWork() { - BigInteger previousWork = this.Previous == null ? BigInteger.Zero : new BigInteger(this.Previous.chainWork); - this.chainWork = previousWork.Add(this.GetBlockTarget()).ToByteArray(); + BigInteger previousWork = this.Previous == null ? BigInteger.Zero : new BigInteger(this.Previous.ChainWorkBytes); + this.ChainWorkBytes = previousWork.Add(this.GetBlockTarget()).ToByteArray(); } /// Calculates the amount of work that this block contributes to the total chain work. diff --git a/src/NBitcoin/MemoryHeaderStore.cs b/src/NBitcoin/MemoryHeaderStore.cs deleted file mode 100644 index 42c98b2d2..000000000 --- a/src/NBitcoin/MemoryHeaderStore.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Collections.Generic; -using NBitcoin.BouncyCastle.Math; -using NBitcoin.Crypto; - -namespace NBitcoin -{ - public interface IBlockHeaderStore - { - BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash); - - bool StoreHeader(BlockHeader blockHeader); - } - - public class MemoryHeaderStore : IBlockHeaderStore - { - private readonly ConcurrentDictionary headers; - - public MemoryHeaderStore() - { - this.headers = new ConcurrentDictionary(); - } - - public BlockHeader GetHeader(ChainedHeader chainedHeader, uint256 hash) - { - if (!this.headers.TryGetValue(hash, out BlockHeader header)) - { - throw new ApplicationException("Header must exist if requested"); - } - - return header; - } - - public bool StoreHeader(BlockHeader blockHeader) - { - return this.headers.TryAdd(blockHeader.GetHash(), blockHeader); - } - } -} \ No newline at end of file