diff --git a/src/Blockcore.Features.BlockStore.Tests/BlockRepositoryTests.cs b/src/Blockcore.Features.BlockStore.Tests/BlockRepositoryTests.cs index a017ba027..f2f37c23a 100644 --- a/src/Blockcore.Features.BlockStore.Tests/BlockRepositoryTests.cs +++ b/src/Blockcore.Features.BlockStore.Tests/BlockRepositoryTests.cs @@ -1,9 +1,11 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using Blockcore.Tests.Common.Logging; using Blockcore.Utilities; using DBreeze; using DBreeze.DataTypes; +using LevelDB; using NBitcoin; using Xunit; @@ -19,15 +21,13 @@ public void InitializesGenesisBlockAndTxIndexOnFirstLoad() { } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); + byte[] blockRow = engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[0])); + bool txIndexRow = BitConverter.ToBoolean(engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[1]))); - Row blockRow = transaction.Select("Common", new byte[0]); - Row txIndexRow = transaction.Select("Common", new byte[1]); - - Assert.Equal(this.Network.GetGenesis().GetHash(), this.DBreezeSerializer.Deserialize(blockRow.Value).Hash); - Assert.False(txIndexRow.Value); + Assert.Equal(this.Network.GetGenesis().GetHash(), this.DBreezeSerializer.Deserialize(blockRow).Hash); + Assert.False(txIndexRow); } } @@ -36,28 +36,23 @@ public void DoesNotOverwriteExistingBlockAndTxIndexOnFirstLoad() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(56), 1))); - transaction.Insert("Common", new byte[1], true); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(56), 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) { } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - - Row blockRow = transaction.Select("Common", new byte[0]); - Row txIndexRow = transaction.Select("Common", new byte[1]); + byte[] blockRow = engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[0])); + bool txIndexRow = BitConverter.ToBoolean(engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[1]))); - Assert.Equal(new HashHeightPair(new uint256(56), 1), this.DBreezeSerializer.Deserialize(blockRow.Value)); - Assert.True(txIndexRow.Value); + Assert.Equal(new HashHeightPair(new uint256(56), 1), this.DBreezeSerializer.Deserialize(blockRow)); + Assert.True(txIndexRow); } } @@ -66,13 +61,10 @@ public void GetTrxAsyncWithoutTransactionIndexReturnsNewTransaction() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - transaction.Insert("Common", new byte[1], false); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(false)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -86,13 +78,11 @@ public void GetTrxAsyncWithoutTransactionInIndexReturnsNull() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); var blockId = new uint256(8920); - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - transaction.Insert("Common", new byte[1], true); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -108,18 +98,16 @@ public void GetTrxAsyncWithTransactionReturnsExistingTransaction() Transaction trans = this.Network.CreateTransaction(); trans.Version = 125; - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { Block block = this.Network.CreateBlock(); block.Header.GetHash(); block.Transactions.Add(trans); - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Block", block.Header.GetHash().ToBytes(), block.ToBytes()); - transaction.Insert("Transaction", trans.GetHash().ToBytes(), block.Header.GetHash().ToBytes()); - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - transaction.Insert("Common", new byte[1], true); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, block.Header.GetHash().ToBytes()), block.ToBytes()); + engine.Put(DBH.Key(BlockRepository.TransactionTableName, trans.GetHash().ToBytes()), block.Header.GetHash().ToBytes()); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -133,12 +121,10 @@ public void GetTrxBlockIdAsyncWithoutTxIndexReturnsDefaultId() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - transaction.Insert("Common", new byte[1], false); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(false)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -152,12 +138,10 @@ public void GetTrxBlockIdAsyncWithoutExistingTransactionReturnsNull() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - transaction.Insert("Common", new byte[1], true); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -171,13 +155,11 @@ public void GetTrxBlockIdAsyncWithTransactionReturnsBlockId() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Transaction", new uint256(26).ToBytes(), new uint256(42).ToBytes()); - transaction.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - transaction.Insert("Common", new byte[1], true); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.TransactionTableName, new uint256(26).ToBytes()), new uint256(42).ToBytes()); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -211,12 +193,10 @@ public void PutAsyncWritesBlocksAndTransactionsToDbAndSavesNextBlockHash() block2.Transactions.Add(transaction); blocks.Add(block2); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction trans = engine.GetTransaction(); - trans.Insert("Common", new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); - trans.Insert("Common", new byte[1], true); - trans.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(uint256.Zero, 1))); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -224,15 +204,14 @@ public void PutAsyncWritesBlocksAndTransactionsToDbAndSavesNextBlockHash() repository.PutBlocks(new HashHeightPair(nextBlockHash, 100), blocks); } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction trans = engine.GetTransaction(); + byte[] blockHashKeyRow = engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[0])); - Row blockHashKeyRow = trans.Select("Common", new byte[0]); - Dictionary blockDict = trans.SelectDictionary("Block"); - Dictionary transDict = trans.SelectDictionary("Transaction"); + Dictionary blockDict = engine.SelectDictionary(BlockRepository.BlockTableName); + Dictionary transDict = engine.SelectDictionary(BlockRepository.TransactionTableName); - Assert.Equal(new HashHeightPair(nextBlockHash, 100), this.DBreezeSerializer.Deserialize(blockHashKeyRow.Value)); + Assert.Equal(new HashHeightPair(nextBlockHash, 100), this.DBreezeSerializer.Deserialize(blockHashKeyRow)); Assert.Equal(2, blockDict.Count); Assert.Equal(3, transDict.Count); @@ -254,11 +233,9 @@ public void PutAsyncWritesBlocksAndTransactionsToDbAndSavesNextBlockHash() public void SetTxIndexUpdatesTxIndex() { string dir = CreateTestDir(this); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction trans = engine.GetTransaction(); - trans.Insert("Common", new byte[1], true); - trans.Commit(); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -266,12 +243,10 @@ public void SetTxIndexUpdatesTxIndex() repository.SetTxIndex(false); } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction trans = engine.GetTransaction(); - - Row txIndexRow = trans.Select("Common", new byte[1]); - Assert.False(txIndexRow.Value); + bool txIndexRow = BitConverter.ToBoolean(engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[1]))); + Assert.False(txIndexRow); } } @@ -281,11 +256,9 @@ public void GetAsyncWithExistingBlockReturnsBlock() string dir = CreateTestDir(this); Block block = this.Network.Consensus.ConsensusFactory.CreateBlock(); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Block", block.GetHash().ToBytes(), block.ToBytes()); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes()); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -307,12 +280,10 @@ public void GetAsyncWithExistingBlocksReturnsBlocks() blocks[i].Header.HashPrevBlock = blocks[i - 1].Header.GetHash(); } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); for (int i = 0; i < blocks.Length; i++) - transaction.Insert("Block", blocks[i].GetHash().ToBytes(), blocks[i].ToBytes()); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, blocks[i].GetHash().ToBytes()), blocks[i].ToBytes()); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -342,11 +313,9 @@ public void ExistAsyncWithExistingBlockReturnsTrue() string dir = CreateTestDir(this); Block block = this.Network.Consensus.ConsensusFactory.CreateBlock(); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Block", block.GetHash().ToBytes(), block.ToBytes()); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes()); } using (IBlockRepository repository = this.SetupRepository(this.Network, dir)) @@ -373,13 +342,11 @@ public void DeleteAsyncRemovesBlocksAndTransactions() Block block = this.Network.CreateBlock(); block.Transactions.Add(this.Network.CreateTransaction()); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction transaction = engine.GetTransaction(); - transaction.Insert("Block", block.GetHash().ToBytes(), block.ToBytes()); - transaction.Insert("Transaction", block.Transactions[0].GetHash().ToBytes(), block.GetHash().ToBytes()); - transaction.Insert("Common", new byte[1], true); - transaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes()); + engine.Put(DBH.Key(BlockRepository.TransactionTableName, block.Transactions[0].GetHash().ToBytes()), block.GetHash().ToBytes()); + engine.Put(DBH.Key(BlockRepository.CommonTableName, new byte[1]), BitConverter.GetBytes(true)); } var tip = new HashHeightPair(new uint256(45), 100); @@ -389,15 +356,13 @@ public void DeleteAsyncRemovesBlocksAndTransactions() repository.Delete(tip, new List { block.GetHash() }); } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction trans = engine.GetTransaction(); - - Row blockHashKeyRow = trans.Select("Common", new byte[0]); - Dictionary blockDict = trans.SelectDictionary("Block"); - Dictionary transDict = trans.SelectDictionary("Transaction"); + byte[] blockHashKeyRow = engine.Get(DBH.Key(BlockRepository.CommonTableName, new byte[0])); + Dictionary blockDict = engine.SelectDictionary(BlockRepository.BlockTableName); + Dictionary transDict = engine.SelectDictionary(BlockRepository.TransactionTableName); - Assert.Equal(tip, this.DBreezeSerializer.Deserialize(blockHashKeyRow.Value)); + Assert.Equal(tip, this.DBreezeSerializer.Deserialize(blockHashKeyRow)); Assert.Empty(blockDict); Assert.Empty(transDict); } @@ -412,11 +377,9 @@ public void ReIndexAsync_TxIndex_OffToOn() block.Transactions.Add(transaction); // Set up database to mimic that created when TxIndex was off. No transactions stored. - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction dbreezeTransaction = engine.GetTransaction(); - dbreezeTransaction.Insert("Block", block.GetHash().ToBytes(), block.ToBytes()); - dbreezeTransaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes()); } // Turn TxIndex on and then reindex database, as would happen on node startup if -txindex and -reindex are set. @@ -427,11 +390,10 @@ public void ReIndexAsync_TxIndex_OffToOn() } // Check that after indexing database, the transaction inside the block is now indexed. - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction dbreezeTransaction = engine.GetTransaction(); - Dictionary blockDict = dbreezeTransaction.SelectDictionary("Block"); - Dictionary transDict = dbreezeTransaction.SelectDictionary("Transaction"); + Dictionary blockDict = engine.SelectDictionary(BlockRepository.BlockTableName); + Dictionary transDict = engine.SelectDictionary(BlockRepository.TransactionTableName); // Block stored as expected. Assert.Single(blockDict); @@ -454,12 +416,10 @@ public void ReIndexAsync_TxIndex_OnToOff() block.Transactions.Add(transaction); // Set up database to mimic that created when TxIndex was on. Transaction from block is stored. - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction dbreezeTransaction = engine.GetTransaction(); - dbreezeTransaction.Insert("Block", block.GetHash().ToBytes(), block.ToBytes()); - dbreezeTransaction.Insert("Transaction", transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); - dbreezeTransaction.Commit(); + engine.Put(DBH.Key(BlockRepository.BlockTableName, block.GetHash().ToBytes()), block.ToBytes()); + engine.Put(DBH.Key(BlockRepository.TransactionTableName, transaction.GetHash().ToBytes()), block.GetHash().ToBytes()); } // Turn TxIndex off and then reindex database, as would happen on node startup if -txindex=0 and -reindex are set. @@ -470,11 +430,10 @@ public void ReIndexAsync_TxIndex_OnToOff() } // Check that after indexing database, the transaction is no longer stored. - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, dir)) { - DBreeze.Transactions.Transaction dbreezeTransaction = engine.GetTransaction(); - Dictionary blockDict = dbreezeTransaction.SelectDictionary("Block"); - Dictionary transDict = dbreezeTransaction.SelectDictionary("Transaction"); + Dictionary blockDict = engine.SelectDictionary(BlockRepository.BlockTableName); + Dictionary transDict = engine.SelectDictionary(BlockRepository.TransactionTableName); // Block still stored as expected. Assert.Single(blockDict); diff --git a/src/Blockcore.Features.BlockStore/BlockRepository.cs b/src/Blockcore.Features.BlockStore/BlockRepository.cs index 3f48e5789..7f694ad86 100644 --- a/src/Blockcore.Features.BlockStore/BlockRepository.cs +++ b/src/Blockcore.Features.BlockStore/BlockRepository.cs @@ -6,12 +6,11 @@ using Blockcore.Configuration; using Blockcore.Interfaces; using Blockcore.Utilities; -using DBreeze; -using DBreeze.DataTypes; -using DBreeze.Exceptions; using DBreeze.Utils; +using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; +using System; namespace Blockcore.Features.BlockStore { @@ -21,7 +20,7 @@ namespace Blockcore.Features.BlockStore public interface IBlockRepository : IBlockStore { /// The dbreeze database engine. - DBreezeEngine DBreeze { get; } + DB Leveldb { get; } /// /// Deletes blocks and indexes for transactions that belong to deleted blocks. @@ -84,13 +83,13 @@ public interface IBlockRepository : IBlockStore public class BlockRepository : IBlockRepository { - internal const string BlockTableName = "Block"; + internal static readonly byte BlockTableName = 1; + internal static readonly byte CommonTableName = 2; + internal static readonly byte TransactionTableName = 3; - internal const string CommonTableName = "Common"; + private readonly DB leveldb; - internal const string TransactionTableName = "Transaction"; - - public DBreezeEngine DBreeze { get; } + private object locker; private readonly ILogger logger; @@ -106,6 +105,8 @@ public class BlockRepository : IBlockRepository /// public bool TxIndex { get; private set; } + public DB Leveldb => this.leveldb; + private readonly DBreezeSerializer dBreezeSerializer; private readonly IReadOnlyDictionary genesisTransactions; @@ -121,7 +122,9 @@ public BlockRepository(Network network, string folder, ILoggerFactory loggerFact Guard.NotEmpty(folder, nameof(folder)); Directory.CreateDirectory(folder); - this.DBreeze = new DBreezeEngine(folder); + var options = new Options { CreateIfMissing = true }; + this.leveldb = new DB(options, folder); + this.locker = new object(); this.logger = loggerFactory.CreateLogger(this.GetType().FullName); this.network = network; @@ -134,23 +137,17 @@ public virtual void Initialize() { Block genesis = this.network.GetGenesis(); - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - bool doCommit = false; - - if (this.LoadTipHashAndHeight(transaction) == null) + if (this.LoadTipHashAndHeight() == null) { - this.SaveTipHashAndHeight(transaction, new HashHeightPair(genesis.GetHash(), 0)); - doCommit = true; + this.SaveTipHashAndHeight(new HashHeightPair(genesis.GetHash(), 0)); } - if (this.LoadTxIndex(transaction) == null) + if (this.LoadTxIndex() == null) { - this.SaveTxIndex(transaction, false); - doCommit = true; + this.SaveTxIndex(false); } - - if (doCommit) transaction.Commit(); } } @@ -171,22 +168,21 @@ public Transaction GetTransactionById(uint256 trxid) } Transaction res = null; - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.ValuesLazyLoadingIsOn = false; + byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxid.ToBytes())); - Row transactionRow = transaction.Select(TransactionTableName, trxid.ToBytes()); - if (!transactionRow.Exists) + if (transactionRow == null) { this.logger.LogTrace("(-)[NO_BLOCK]:null"); return null; } - Row blockRow = transaction.Select(BlockTableName, transactionRow.Value); + byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, transactionRow)); - if (blockRow.Exists) + if (blockRow != null) { - var block = this.dBreezeSerializer.Deserialize(blockRow.Value); + var block = this.dBreezeSerializer.Deserialize(blockRow); res = block.Transactions.FirstOrDefault(t => t.GetHash() == trxid); } } @@ -205,10 +201,8 @@ public Transaction GetTransactionById(uint256 trxid) Transaction[] txes = new Transaction[trxids.Length]; - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.ValuesLazyLoadingIsOn = false; - for (int i = 0; i < trxids.Length; i++) { cancellation.ThrowIfCancellationRequested(); @@ -229,22 +223,22 @@ public Transaction GetTransactionById(uint256 trxid) continue; } - Row transactionRow = transaction.Select(TransactionTableName, trxids[i].ToBytes()); - if (!transactionRow.Exists) + byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxids[i].ToBytes())); + if (transactionRow == null) { this.logger.LogTrace("(-)[NO_TX_ROW]:null"); return null; } - Row blockRow = transaction.Select(BlockTableName, transactionRow.Value); + byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, transactionRow)); - if (!blockRow.Exists) + if (blockRow != null) { this.logger.LogTrace("(-)[NO_BLOCK]:null"); return null; } - var block = this.dBreezeSerializer.Deserialize(blockRow.Value); + var block = this.dBreezeSerializer.Deserialize(blockRow); Transaction tx = block.Transactions.FirstOrDefault(t => t.GetHash() == trxids[i]); txes[i] = tx; @@ -271,19 +265,17 @@ public uint256 GetBlockIdByTransactionId(uint256 trxid) } uint256 res = null; - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.ValuesLazyLoadingIsOn = false; - - Row transactionRow = transaction.Select(TransactionTableName, trxid.ToBytes()); - if (transactionRow.Exists) - res = new uint256(transactionRow.Value); + byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxid.ToBytes())); + if (transactionRow != null) + res = new uint256(transactionRow); } return res; } - protected virtual void OnInsertBlocks(DBreeze.Transactions.Transaction dbreezeTransaction, List blocks) + protected virtual void OnInsertBlocks(List blocks) { var transactions = new List<(Transaction, Block)>(); var byteListComparer = new ByteListComparer(); @@ -300,50 +292,58 @@ protected virtual void OnInsertBlocks(DBreeze.Transactions.Transaction dbreezeTr List> blockList = blockDict.ToList(); blockList.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Key.ToBytes(), pair2.Key.ToBytes())); - // Index blocks. - foreach (KeyValuePair kv in blockList) + using (var batch = new WriteBatch()) { - uint256 blockId = kv.Key; - Block block = kv.Value; - - // If the block is already in store don't write it again. - Row blockRow = dbreezeTransaction.Select(BlockTableName, blockId.ToBytes()); - if (!blockRow.Exists) + // Index blocks. + foreach (KeyValuePair kv in blockList) { - dbreezeTransaction.Insert(BlockTableName, blockId.ToBytes(), this.dBreezeSerializer.Serialize(block)); + uint256 blockId = kv.Key; + Block block = kv.Value; - if (this.TxIndex) + // If the block is already in store don't write it again. + byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, blockId.ToBytes())); + if (blockRow == null) { - foreach (Transaction transaction in block.Transactions) - transactions.Add((transaction, block)); + batch.Put(DBH.Key(BlockTableName, blockId.ToBytes()), this.dBreezeSerializer.Serialize(block)); + + if (this.TxIndex) + { + foreach (Transaction transaction in block.Transactions) + transactions.Add((transaction, block)); + } } } + + this.leveldb.Write(batch); } if (this.TxIndex) - this.OnInsertTransactions(dbreezeTransaction, transactions); + this.OnInsertTransactions(transactions); } - protected virtual void OnInsertTransactions(DBreeze.Transactions.Transaction dbreezeTransaction, List<(Transaction, Block)> transactions) + protected virtual void OnInsertTransactions(List<(Transaction, Block)> transactions) { var byteListComparer = new ByteListComparer(); transactions.Sort((pair1, pair2) => byteListComparer.Compare(pair1.Item1.GetHash().ToBytes(), pair2.Item1.GetHash().ToBytes())); - // Index transactions. - foreach ((Transaction transaction, Block block) in transactions) - dbreezeTransaction.Insert(TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); + using (var batch = new WriteBatch()) + { + // Index transactions. + foreach ((Transaction transaction, Block block) in transactions) + batch.Put(DBH.Key(TransactionTableName, transaction.GetHash().ToBytes()), block.GetHash().ToBytes()); + + this.leveldb.Write(batch); + } } public IEnumerable EnumeratehBatch(List headers) { - using (DBreeze.Transactions.Transaction dbreezeTransaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - dbreezeTransaction.SynchronizeTables(BlockTableName, TransactionTableName); - foreach (ChainedHeader chainedHeader in headers) { - Row blockRow = dbreezeTransaction.Select(BlockTableName, chainedHeader.HashBlock.ToBytes()); - Block block = blockRow.Exists ? this.dBreezeSerializer.Deserialize(blockRow.Value) : null; + byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, chainedHeader.HashBlock.ToBytes())); + Block block = blockRow != null ? this.dBreezeSerializer.Deserialize(blockRow) : null; yield return block; } } @@ -352,16 +352,14 @@ public IEnumerable EnumeratehBatch(List headers) /// public void ReIndex() { - using (DBreeze.Transactions.Transaction dbreezeTransaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - dbreezeTransaction.SynchronizeTables(BlockTableName, TransactionTableName); - if (this.TxIndex) { int rowCount = 0; // Insert transactions to database. - var totalBlocksCount = dbreezeTransaction.Count(BlockTableName); + int totalBlocksCount = this.TipHashAndHeight?.Height ?? 0; var warningMessage = new StringBuilder(); warningMessage.AppendLine("".PadRight(59, '=') + " W A R N I N G " + "".PadRight(59, '=')); @@ -373,32 +371,42 @@ public void ReIndex() warningMessage.AppendLine(); this.logger.LogInformation(warningMessage.ToString()); - - IEnumerable> blockRows = dbreezeTransaction.SelectForward(BlockTableName); - foreach (Row blockRow in blockRows) + using (var batch = new WriteBatch()) { - var block = this.dBreezeSerializer.Deserialize(blockRow.Value); - foreach (Transaction transaction in block.Transactions) + var enumerator = this.leveldb.GetEnumerator(); + while (enumerator.MoveNext()) { - dbreezeTransaction.Insert(TransactionTableName, transaction.GetHash().ToBytes(), block.GetHash().ToBytes()); + if (enumerator.Current.Key[0] == BlockTableName) + { + var block = this.dBreezeSerializer.Deserialize(enumerator.Current.Value); + foreach (Transaction transaction in block.Transactions) + { + batch.Put(DBH.Key(TransactionTableName, transaction.GetHash().ToBytes()), block.GetHash().ToBytes()); + } + + // inform the user about the ongoing operation + if (++rowCount % 1000 == 0) + { + this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); + } + } } - // inform the user about the ongoing operation - if (++rowCount % 1000 == 0) - { - this.logger.LogInformation("Reindex in process... {0}/{1} blocks processed.", rowCount, totalBlocksCount); - } + this.leveldb.Write(batch); } this.logger.LogInformation("Reindex completed successfully."); } else { - // Clear tx from database. - dbreezeTransaction.RemoveAllKeys(TransactionTableName, true); + var enumerator = this.leveldb.GetEnumerator(); + while (enumerator.MoveNext()) + { + // Clear tx from database. + if (enumerator.Current.Key[0] == TransactionTableName) + this.leveldb.Delete(enumerator.Current.Key); + } } - - dbreezeTransaction.Commit(); } } @@ -410,66 +418,59 @@ public void PutBlocks(HashHeightPair newTip, List blocks) // DBreeze is faster if sort ascending by key in memory before insert // however we need to find how byte arrays are sorted in DBreeze. - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.SynchronizeTables(BlockTableName, TransactionTableName, CommonTableName); - this.OnInsertBlocks(transaction, blocks); + this.OnInsertBlocks(blocks); // Commit additions - this.SaveTipHashAndHeight(transaction, newTip); - transaction.Commit(); + this.SaveTipHashAndHeight(newTip); } } - private bool? LoadTxIndex(DBreeze.Transactions.Transaction dbreezeTransaction) + private bool? LoadTxIndex() { bool? res = null; - Row row = dbreezeTransaction.Select(CommonTableName, TxIndexKey); - if (row.Exists) + byte[] row = this.leveldb.Get(DBH.Key(CommonTableName, TxIndexKey)); + if (row != null) { - this.TxIndex = row.Value; - res = row.Value; + this.TxIndex = BitConverter.ToBoolean(row); + res = this.TxIndex; } return res; } - private void SaveTxIndex(DBreeze.Transactions.Transaction dbreezeTransaction, bool txIndex) + private void SaveTxIndex(bool txIndex) { this.TxIndex = txIndex; - dbreezeTransaction.Insert(CommonTableName, TxIndexKey, txIndex); + this.leveldb.Put(DBH.Key(CommonTableName, TxIndexKey), BitConverter.GetBytes(txIndex)); } /// public void SetTxIndex(bool txIndex) { - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - this.SaveTxIndex(transaction, txIndex); - transaction.Commit(); + this.SaveTxIndex(txIndex); } } - private HashHeightPair LoadTipHashAndHeight(DBreeze.Transactions.Transaction dbreezeTransaction) + private HashHeightPair LoadTipHashAndHeight() { if (this.TipHashAndHeight == null) { - dbreezeTransaction.ValuesLazyLoadingIsOn = false; - - Row row = dbreezeTransaction.Select(CommonTableName, RepositoryTipKey); - if (row.Exists) - this.TipHashAndHeight = this.dBreezeSerializer.Deserialize(row.Value); - - dbreezeTransaction.ValuesLazyLoadingIsOn = true; + byte[] row = this.leveldb.Get(DBH.Key(CommonTableName, RepositoryTipKey)); + if (row != null) + this.TipHashAndHeight = this.dBreezeSerializer.Deserialize(row); } return this.TipHashAndHeight; } - private void SaveTipHashAndHeight(DBreeze.Transactions.Transaction dbreezeTransaction, HashHeightPair newTip) + private void SaveTipHashAndHeight(HashHeightPair newTip) { this.TipHashAndHeight = newTip; - dbreezeTransaction.Insert(CommonTableName, RepositoryTipKey, this.dBreezeSerializer.Serialize(newTip)); + this.leveldb.Put(DBH.Key(CommonTableName, RepositoryTipKey), this.dBreezeSerializer.Serialize(newTip)); } /// @@ -478,11 +479,9 @@ public Block GetBlock(uint256 hash) Guard.NotNull(hash, nameof(hash)); Block res = null; - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.ValuesLazyLoadingIsOn = false; - - var results = this.GetBlocksFromHashes(transaction, new List {hash}); + var results = this.GetBlocksFromHashes(new List { hash }); if (results.FirstOrDefault() != null) res = results.FirstOrDefault(); @@ -498,11 +497,9 @@ public List GetBlocks(List hashes) List blocks; - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.ValuesLazyLoadingIsOn = false; - - blocks = this.GetBlocksFromHashes(transaction, hashes); + blocks = this.GetBlocksFromHashes(hashes); } return blocks; @@ -514,25 +511,25 @@ public bool Exist(uint256 hash) Guard.NotNull(hash, nameof(hash)); bool res = false; - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { // Lazy loading is on so we don't fetch the whole value, just the row. byte[] key = hash.ToBytes(); - Row blockRow = transaction.Select("Block", key); - if (blockRow.Exists) + byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, key)); + if (blockRow != null) res = true; } return res; } - protected virtual void OnDeleteTransactions(DBreeze.Transactions.Transaction dbreezeTransaction, List<(Transaction, Block)> transactions) + protected virtual void OnDeleteTransactions(List<(Transaction, Block)> transactions) { foreach ((Transaction transaction, Block block) in transactions) - dbreezeTransaction.RemoveKey(TransactionTableName, transaction.GetHash().ToBytes()); + this.leveldb.Delete(DBH.Key(TransactionTableName, transaction.GetHash().ToBytes())); } - protected virtual void OnDeleteBlocks(DBreeze.Transactions.Transaction dbreezeTransaction, List blocks) + protected virtual void OnDeleteBlocks(List blocks) { if (this.TxIndex) { @@ -542,14 +539,14 @@ protected virtual void OnDeleteBlocks(DBreeze.Transactions.Transaction dbreezeTr foreach (Transaction transaction in block.Transactions) transactions.Add((transaction, block)); - this.OnDeleteTransactions(dbreezeTransaction, transactions); + this.OnDeleteTransactions(transactions); } foreach (Block block in blocks) - dbreezeTransaction.RemoveKey(BlockTableName, block.GetHash().ToBytes()); + this.leveldb.Delete(DBH.Key(BlockTableName, block.GetHash().ToBytes())); } - public List GetBlocksFromHashes(DBreeze.Transactions.Transaction dbreezeTransaction, List hashes) + public List GetBlocksFromHashes(List hashes) { var results = new Dictionary(); @@ -568,10 +565,10 @@ public List GetBlocksFromHashes(DBreeze.Transactions.Transaction dbreezeT continue; } - Row blockRow = dbreezeTransaction.Select(BlockTableName, key.Item2); - if (blockRow.Exists) + byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, key.Item2)); + if (blockRow != null) { - results[key.Item1] = this.dBreezeSerializer.Deserialize(blockRow.Value); + results[key.Item1] = this.dBreezeSerializer.Deserialize(blockRow); this.logger.LogDebug("Block hash '{0}' loaded from the store.", key.Item1); } @@ -593,15 +590,11 @@ public void Delete(HashHeightPair newTip, List hashes) Guard.NotNull(newTip, nameof(newTip)); Guard.NotNull(hashes, nameof(hashes)); - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.SynchronizeTables(BlockTableName, CommonTableName, TransactionTableName); - transaction.ValuesLazyLoadingIsOn = false; - - List blocks = this.GetBlocksFromHashes(transaction, hashes); - this.OnDeleteBlocks(transaction, blocks.Where(b => b != null).ToList()); - this.SaveTipHashAndHeight(transaction, newTip); - transaction.Commit(); + List blocks = this.GetBlocksFromHashes(hashes); + this.OnDeleteBlocks(blocks.Where(b => b != null).ToList()); + this.SaveTipHashAndHeight(newTip); } } @@ -610,23 +603,18 @@ public void DeleteBlocks(List hashes) { Guard.NotNull(hashes, nameof(hashes)); - using (DBreeze.Transactions.Transaction transaction = this.DBreeze.GetTransaction()) + lock (this.locker) { - transaction.SynchronizeTables(BlockRepository.BlockTableName, BlockRepository.CommonTableName, BlockRepository.TransactionTableName); - transaction.ValuesLazyLoadingIsOn = false; - - List blocks = this.GetBlocksFromHashes(transaction, hashes); - - this.OnDeleteBlocks(transaction, blocks.Where(b => b != null).ToList()); + List blocks = this.GetBlocksFromHashes(hashes); - transaction.Commit(); + this.OnDeleteBlocks(blocks.Where(b => b != null).ToList()); } } /// public void Dispose() { - this.DBreeze.Dispose(); + this.leveldb.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Features.BlockStore/Pruning/PrunedBlockRepository.cs b/src/Blockcore.Features.BlockStore/Pruning/PrunedBlockRepository.cs index 17489059f..034a6016c 100644 --- a/src/Blockcore.Features.BlockStore/Pruning/PrunedBlockRepository.cs +++ b/src/Blockcore.Features.BlockStore/Pruning/PrunedBlockRepository.cs @@ -3,6 +3,7 @@ using System.Threading.Tasks; using Blockcore.Utilities; using DBreeze.DataTypes; +using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; @@ -31,10 +32,7 @@ public PrunedBlockRepository(IBlockRepository blockRepository, DBreezeSerializer /// public void Initialize() { - using (DBreeze.Transactions.Transaction transaction = this.blockRepository.DBreeze.GetTransaction()) - { - this.LoadPrunedTip(transaction); - } + this.LoadPrunedTip(this.blockRepository.Leveldb); } /// @@ -48,11 +46,7 @@ public void PruneAndCompactDatabase(ChainedHeader blockRepositoryTip, Network ne this.PrunedTip = new HashHeightPair(genesis.GetHash(), 0); - using (DBreeze.Transactions.Transaction transaction = this.blockRepository.DBreeze.GetTransaction()) - { - transaction.Insert(BlockRepository.CommonTableName, prunedTipKey, this.dBreezeSerializer.Serialize(this.PrunedTip)); - transaction.Commit(); - } + this.blockRepository.Leveldb.Put(DBH.Key(BlockRepository.CommonTableName, prunedTipKey), this.dBreezeSerializer.Serialize(this.PrunedTip)); } if (nodeInitializing) @@ -110,17 +104,13 @@ private void PrepareDatabaseForCompacting(ChainedHeader blockRepositoryTip) this.UpdatePrunedTip(blockRepositoryTip.GetAncestor(upperHeight)); } - private void LoadPrunedTip(DBreeze.Transactions.Transaction dbreezeTransaction) + private void LoadPrunedTip(DB leveldb) { if (this.PrunedTip == null) { - dbreezeTransaction.ValuesLazyLoadingIsOn = false; - - Row row = dbreezeTransaction.Select(BlockRepository.CommonTableName, prunedTipKey); - if (row.Exists) - this.PrunedTip = this.dBreezeSerializer.Deserialize(row.Value); - - dbreezeTransaction.ValuesLazyLoadingIsOn = true; + byte[] row = leveldb.Get(DBH.Key(BlockRepository.CommonTableName, prunedTipKey)); + if (row != null) + this.PrunedTip = this.dBreezeSerializer.Deserialize(row); } } @@ -131,34 +121,6 @@ private void CompactDataBase() { Task task = Task.Run(() => { - using (DBreeze.Transactions.Transaction dbreezeTransaction = this.blockRepository.DBreeze.GetTransaction()) - { - dbreezeTransaction.SynchronizeTables(BlockRepository.BlockTableName, BlockRepository.TransactionTableName); - - var tempBlocks = dbreezeTransaction.SelectDictionary(BlockRepository.BlockTableName); - - if (tempBlocks.Count != 0) - { - this.logger.LogInformation($"{tempBlocks.Count} blocks will be copied to the pruned table."); - - dbreezeTransaction.RemoveAllKeys(BlockRepository.BlockTableName, true); - dbreezeTransaction.InsertDictionary(BlockRepository.BlockTableName, tempBlocks, false); - - var tempTransactions = dbreezeTransaction.SelectDictionary(BlockRepository.TransactionTableName); - if (tempTransactions.Count != 0) - { - this.logger.LogInformation($"{tempTransactions.Count} transactions will be copied to the pruned table."); - dbreezeTransaction.RemoveAllKeys(BlockRepository.TransactionTableName, true); - dbreezeTransaction.InsertDictionary(BlockRepository.TransactionTableName, tempTransactions, false); - } - - // Save the hash and height of where the node was pruned up to. - dbreezeTransaction.Insert(BlockRepository.CommonTableName, prunedTipKey, this.dBreezeSerializer.Serialize(this.PrunedTip)); - } - - dbreezeTransaction.Commit(); - } - return Task.CompletedTask; }); } @@ -169,4 +131,4 @@ public void UpdatePrunedTip(ChainedHeader tip) this.PrunedTip = new HashHeightPair(tip); } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs b/src/Blockcore.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs index 1dd78b64a..e85c8379e 100644 --- a/src/Blockcore.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs +++ b/src/Blockcore.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderRepositoryTests.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; using Blockcore.Features.Consensus.ProvenBlockHeaders; @@ -9,6 +10,7 @@ using DBreeze; using DBreeze.Utils; using FluentAssertions; +using LevelDB; using Microsoft.Extensions.Logging; using Moq; using NBitcoin; @@ -20,8 +22,8 @@ public class ProvenBlockHeaderRepositoryTests : LogsTestBase { private readonly Mock loggerFactory; private readonly DBreezeSerializer dBreezeSerializer; - private const string ProvenBlockHeaderTable = "ProvenBlockHeader"; - private const string BlockHashTable = "BlockHashHeight"; + private static readonly byte ProvenBlockHeaderTable = 1; + private static readonly byte BlockHashHeightTable = 2; public ProvenBlockHeaderRepositoryTests() : base(KnownNetworks.StratisTest) { @@ -58,14 +60,10 @@ public async Task PutAsync_WritesProvenBlockHeaderAndSavesBlockHashAsync() await repo.PutAsync(items, blockHashHeightPair); } - using (var engine = new DBreezeEngine(folder)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) { - DBreeze.Transactions.Transaction txn = engine.GetTransaction(); - txn.SynchronizeTables(ProvenBlockHeaderTable); - txn.ValuesLazyLoadingIsOn = false; - - var headerOut = this.dBreezeSerializer.Deserialize(txn.Select(ProvenBlockHeaderTable, blockHashHeightPair.Height.ToBytes()).Value); - var hashHeightPairOut = this.DBreezeSerializer.Deserialize(txn.Select(BlockHashTable, new byte[0].ToBytes()).Value); + var headerOut = this.dBreezeSerializer.Deserialize(engine.Get(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHashHeightPair.Height)))); + var hashHeightPairOut = this.DBreezeSerializer.Deserialize(engine.Get(DBH.Key(BlockHashHeightTable, new byte[] { 1 }))); headerOut.Should().NotBeNull(); headerOut.GetHash().Should().Be(provenBlockHeaderIn.GetHash()); @@ -93,13 +91,13 @@ public async Task PutAsync_Inserts_MultipleProvenBlockHeadersAsync() } // Check the ProvenBlockHeader exists in the database. - using (var engine = new DBreezeEngine(folder)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) { - DBreeze.Transactions.Transaction txn = engine.GetTransaction(); - txn.SynchronizeTables(ProvenBlockHeaderTable); - txn.ValuesLazyLoadingIsOn = false; - - var headersOut = txn.SelectDictionary(ProvenBlockHeaderTable); + var headersOut = new Dictionary(); + var enumeator = engine.GetEnumerator(); + while (enumeator.MoveNext()) + if (enumeator.Current.Key[0] == ProvenBlockHeaderTable) + headersOut.Add(enumeator.Current.Key, enumeator.Current.Value); headersOut.Keys.Count.Should().Be(2); this.dBreezeSerializer.Deserialize(headersOut.First().Value).GetHash().Should().Be(items[0].GetHash()); @@ -116,11 +114,9 @@ public async Task GetAsync_ReadsProvenBlockHeaderAsync() int blockHeight = 1; - using (var engine = new DBreezeEngine(folder)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) { - DBreeze.Transactions.Transaction txn = engine.GetTransaction(); - txn.Insert(ProvenBlockHeaderTable, blockHeight.ToBytes(), this.dBreezeSerializer.Serialize(headerIn)); - txn.Commit(); + engine.Put(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(blockHeight)), this.dBreezeSerializer.Serialize(headerIn)); } // Query the repository for the item that was inserted in the above code. @@ -138,12 +134,10 @@ public async Task GetAsync_WithWrongBlockHeightReturnsNullAsync() { string folder = CreateTestDir(this); - using (var engine = new DBreezeEngine(folder)) + using (var engine = new DB(new Options() { CreateIfMissing = true }, folder)) { - DBreeze.Transactions.Transaction txn = engine.GetTransaction(); - txn.Insert(ProvenBlockHeaderTable, 1.ToBytes(), this.dBreezeSerializer.Serialize(CreateNewProvenBlockHeaderMock())); - txn.Insert(BlockHashTable, new byte[0], this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(), 1))); - txn.Commit(); + engine.Put(DBH.Key(ProvenBlockHeaderTable, BitConverter.GetBytes(1)), this.dBreezeSerializer.Serialize(CreateNewProvenBlockHeaderMock())); + engine.Put(DBH.Key(BlockHashHeightTable, new byte[0]), this.DBreezeSerializer.Serialize(new HashHeightPair(new uint256(), 1))); } using (ProvenBlockHeaderRepository repo = this.SetupRepository(this.Network, folder)) @@ -159,7 +153,7 @@ public async Task GetAsync_WithWrongBlockHeightReturnsNullAsync() } [Fact] - public async Task PutAsync_Add_Ten_ProvenBlockHeaders_Dispose_On_Initialise_Repo_TipHeight_Should_Be_At_Last_Saved_TipAsync() + public async Task PutAsync_DisposeOnInitialise_ShouldBeAtLastSavedTipAsync() { string folder = CreateTestDir(this); @@ -195,4 +189,4 @@ private ProvenBlockHeaderRepository SetupRepository(Network network, string fold return repo; } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs b/src/Blockcore.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs index f34bb88c1..b8b1646bd 100644 --- a/src/Blockcore.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs +++ b/src/Blockcore.Features.Consensus/ProvenBlockHeaders/ProvenBlockHeaderRepository.cs @@ -1,13 +1,13 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Threading.Tasks; using Blockcore.Configuration; using Blockcore.Interfaces; using Blockcore.Utilities; -using DBreeze; -using DBreeze.DataTypes; using DBreeze.Utils; +using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; @@ -24,9 +24,11 @@ public class ProvenBlockHeaderRepository : IProvenBlockHeaderRepository private readonly ILogger logger; /// - /// Access to DBreeze database. + /// Access to database. /// - private readonly DBreezeEngine dbreeze; + private readonly DB leveldb; + + private object locker; /// /// Specification of the network the node runs on - RegTest/TestNet/MainNet. @@ -36,13 +38,10 @@ public class ProvenBlockHeaderRepository : IProvenBlockHeaderRepository /// /// Database key under which the block hash and height of a tip is stored. /// - private static readonly byte[] blockHashHeightKey = new byte[0]; + private static readonly byte[] blockHashHeightKey = new byte[] { 1 }; - /// - /// DBreeze table names. - /// - private const string ProvenBlockHeaderTable = "ProvenBlockHeader"; - private const string BlockHashHeightTable = "BlockHashHeight"; + private static readonly byte provenBlockHeaderTable = 1; + private static readonly byte blockHashHeightTable = 2; /// /// Current tip. @@ -85,7 +84,11 @@ public ProvenBlockHeaderRepository(Network network, string folder, ILoggerFactor Directory.CreateDirectory(folder); - this.dbreeze = new DBreezeEngine(folder); + // Open a connection to a new DB and create if not found + var options = new Options { CreateIfMissing = true }; + this.leveldb = new DB(options, folder); + this.locker = new object(); + this.network = network; } @@ -94,21 +97,16 @@ public Task InitializeAsync() { Task task = Task.Run(() => { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) - { - this.TipHashHeight = this.GetTipHash(transaction); - - if (this.TipHashHeight != null) - return; + this.TipHashHeight = this.GetTipHash(); - var hashHeight = new HashHeightPair(this.network.GetGenesis().GetHash(), 0); + if (this.TipHashHeight != null) + return; - this.SetTip(transaction, hashHeight); + var hashHeight = new HashHeightPair(this.network.GetGenesis().GetHash(), 0); - transaction.Commit(); + this.SetTip(hashHeight); - this.TipHashHeight = hashHeight; - } + this.TipHashHeight = hashHeight; }); return task; @@ -119,19 +117,17 @@ public Task GetAsync(int blockHeight) { Task task = Task.Run(() => { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) - { - transaction.SynchronizeTables(ProvenBlockHeaderTable); - - transaction.ValuesLazyLoadingIsOn = false; + byte[] row = null; - Row row = transaction.Select(ProvenBlockHeaderTable, blockHeight.ToBytes()); + lock (this.locker) + { + row = this.leveldb.Get(DBH.Key(provenBlockHeaderTable, BitConverter.GetBytes(blockHeight))); + } - if (row.Exists) - return this.dBreezeSerializer.Deserialize(row.Value); + if (row != null) + return this.dBreezeSerializer.Deserialize(row); - return null; - } + return null; }); return task; @@ -149,18 +145,11 @@ public Task PutAsync(SortedDictionary headers, HashHeigh { this.logger.LogDebug("({0}.Count():{1})", nameof(headers), headers.Count()); - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) - { - transaction.SynchronizeTables(BlockHashHeightTable, ProvenBlockHeaderTable); - - this.InsertHeaders(transaction, headers); - - this.SetTip(transaction, newTip); + this.InsertHeaders(headers); - transaction.Commit(); + this.SetTip(newTip); - this.TipHashHeight = newTip; - } + this.TipHashHeight = newTip; }); return task; @@ -171,22 +160,32 @@ public Task PutAsync(SortedDictionary headers, HashHeigh /// /// Open DBreeze transaction. /// Hash height pair of the new block tip. - private void SetTip(DBreeze.Transactions.Transaction transaction, HashHeightPair newTip) + private void SetTip(HashHeightPair newTip) { Guard.NotNull(newTip, nameof(newTip)); - transaction.Insert(BlockHashHeightTable, blockHashHeightKey, this.dBreezeSerializer.Serialize(newTip)); + lock (this.locker) + { + this.leveldb.Put(DBH.Key(blockHashHeightTable, blockHashHeightKey), this.dBreezeSerializer.Serialize(newTip)); + } } /// /// Inserts items into to the database. /// - /// Open DBreeze transaction. /// List of items to save. - private void InsertHeaders(DBreeze.Transactions.Transaction transaction, SortedDictionary headers) + private void InsertHeaders(SortedDictionary headers) { - foreach (KeyValuePair header in headers) - transaction.Insert(ProvenBlockHeaderTable, header.Key.ToBytes(), this.dBreezeSerializer.Serialize(header.Value)); + using (var batch = new WriteBatch()) + { + foreach (KeyValuePair header in headers) + batch.Put(DBH.Key(provenBlockHeaderTable, BitConverter.GetBytes(header.Key)), this.dBreezeSerializer.Serialize(header.Value)); + + lock (this.locker) + { + this.leveldb.Write(batch); + } + } // Store the latest ProvenBlockHeader in memory. this.provenBlockHeaderTip = headers.Last().Value; @@ -195,16 +194,19 @@ private void InsertHeaders(DBreeze.Transactions.Transaction transaction, SortedD /// /// Retrieves the current tip from disk. /// - /// Open DBreeze transaction. /// Hash of blocks current tip. - private HashHeightPair GetTipHash(DBreeze.Transactions.Transaction transaction) + private HashHeightPair GetTipHash() { HashHeightPair tipHash = null; - Row row = transaction.Select(BlockHashHeightTable, blockHashHeightKey); + byte[] row = null; + lock (this.locker) + { + row = this.leveldb.Get(DBH.Key(blockHashHeightTable, blockHashHeightKey)); + } - if (row.Exists) - tipHash = this.dBreezeSerializer.Deserialize(row.Value); + if (row != null) + tipHash = this.dBreezeSerializer.Deserialize(row); return tipHash; } @@ -212,7 +214,7 @@ private HashHeightPair GetTipHash(DBreeze.Transactions.Transaction transaction) /// public void Dispose() { - this.dbreeze?.Dispose(); + this.leveldb?.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Features.PoA/Voting/VotingManager.cs b/src/Blockcore.Features.PoA/Voting/VotingManager.cs index 83a02ab62..40f58d2eb 100644 --- a/src/Blockcore.Features.PoA/Voting/VotingManager.cs +++ b/src/Blockcore.Features.PoA/Voting/VotingManager.cs @@ -3,6 +3,7 @@ using System.Linq; using System.Text; using Blockcore.Configuration; +using Blockcore.Consensus; using Blockcore.EventBus; using Blockcore.EventBus.CoreEvents; using Blockcore.Primitives; diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip index 8b0777eff..32d9b3c8a 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip index d341fd781..2f3daaffd 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip index 9309257e0..986af81f6 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest100NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip index 4e2de7576..699f659cd 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip index 6c6151f62..7801f0dd8 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip index 392b434c6..e4c7193d5 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest10NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip index 2fd3cd135..2cc99d200 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip index 8a2f5c242..02960ce4a 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip index de109caa3..14b38dd5b 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/RegTest150NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip index d5c36e44b..2c44778e6 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip index 9d39bc087..7ae1f1338 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip index 77136f93f..e49560bea 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest100NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip index 7af0ab277..5d5163f0d 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip index 8515c2ef9..bdd67dfff 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip index 596a9eb6e..19e69e75c 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest10NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip index 14952f41b..739550dec 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Listener.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip index dde0b5f53..5f19b4bc9 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150Miner.zip differ diff --git a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip index 2ce40be9e..ce1ef1321 100644 Binary files a/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip and b/src/Blockcore.IntegrationTests.Common/ReadyData/StratisRegTest150NoWallet.zip differ diff --git a/src/Blockcore.IntegrationTests/GenerateChain.cs b/src/Blockcore.IntegrationTests/GenerateChain.cs index c14afa4b3..d80fa72ee 100644 --- a/src/Blockcore.IntegrationTests/GenerateChain.cs +++ b/src/Blockcore.IntegrationTests/GenerateChain.cs @@ -18,7 +18,7 @@ public class GenerateChain private const string MinerMnemonic = "elevator slight dad hair table forum maze feed trim ignore field mystery"; private const string ListenerMnemonic = "seminar cool use bleak drink section rent bid language obey skin round"; - private const string DataPath = @"..\..\..\..\Stratis.Bitcoin.IntegrationTests.Common\ReadyData"; + private const string DataPath = @"..\..\..\..\Blockcore.IntegrationTests.Common\ReadyData"; public GenerateChain() { diff --git a/src/Blockcore.IntegrationTests/Miners/ProofOfWorkMiningTests.cs b/src/Blockcore.IntegrationTests/Miners/ProofOfWorkMiningTests.cs index 981d6411c..3aeeb7587 100644 --- a/src/Blockcore.IntegrationTests/Miners/ProofOfWorkMiningTests.cs +++ b/src/Blockcore.IntegrationTests/Miners/ProofOfWorkMiningTests.cs @@ -49,7 +49,7 @@ public void MiningAndPropagatingPOW_MineBlockCheckPeerHasNewBlock() } [Fact] - public async Task MiningAndPropagatingPOW_MineBlockNotPushedToConsensusCode_SupercededByBetterBlockOnReorg_InitialBlockRejectedAsync() + public async Task MineBlockNotPushedToConsensusCode_SupercededByBetterBlockOnReorg_InitialBlockRejectedAsync() { using (NodeBuilder builder = NodeBuilder.Create(this)) { @@ -81,4 +81,4 @@ public async Task MiningAndPropagatingPOW_MineBlockNotPushedToConsensusCode_Supe } } } -} +} \ No newline at end of file diff --git a/src/Blockcore.IntegrationTests/Wallet/WalletPostOperationsTests.cs b/src/Blockcore.IntegrationTests/Wallet/WalletPostOperationsTests.cs index 656676343..7d6ccc658 100644 --- a/src/Blockcore.IntegrationTests/Wallet/WalletPostOperationsTests.cs +++ b/src/Blockcore.IntegrationTests/Wallet/WalletPostOperationsTests.cs @@ -728,7 +728,7 @@ public async Task SendingATransactionWithAnOpReturn() var script = opReturnOutputFromBlock.ScriptPubKey.Asm; string[] ops = script.Split(" "); ops[0].Should().Be("OP_RETURN"); - Encoders.Hex.DecodeData(ops[1]).Should().BeEquivalentTo("some data to send".ToBytes()); + Encoders.Hex.DecodeData(ops[1]).Should().BeEquivalentTo(System.Text.Encoding.UTF8.GetBytes("some data to send")); } } @@ -813,4 +813,4 @@ public async Task GetHistoryFromMiningNode() } } } -} +} \ No newline at end of file diff --git a/src/Blockcore.Tests/Base/ChainRepositoryTest.cs b/src/Blockcore.Tests/Base/ChainRepositoryTest.cs index 1a133db10..da3c7330d 100644 --- a/src/Blockcore.Tests/Base/ChainRepositoryTest.cs +++ b/src/Blockcore.Tests/Base/ChainRepositoryTest.cs @@ -5,6 +5,7 @@ using Blockcore.Utilities; using DBreeze; using DBreeze.DataTypes; +using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; using Xunit; @@ -32,12 +33,14 @@ public void SaveWritesChainToDisk() repo.SaveAsync(chain).GetAwaiter().GetResult(); } - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options { CreateIfMissing = true }, dir)) { ChainedHeader tip = null; - foreach (Row row in engine.GetTransaction().SelectForward("Chain")) + var itr = engine.GetEnumerator(); + + while (itr.MoveNext()) { - var blockHeader = this.dBreezeSerializer.Deserialize(row.Value); + var blockHeader = this.dBreezeSerializer.Deserialize(itr.Current.Value); if (tip != null && blockHeader.HashPrevBlock != tip.HashBlock) break; @@ -54,9 +57,9 @@ public void GetChainReturnsConcurrentChainFromDisk() var chain = new ChainIndexer(KnownNetworks.StratisRegTest); ChainedHeader tip = this.AppendBlock(chain); - using (var engine = new DBreezeEngine(dir)) + using (var engine = new DB(new Options { CreateIfMissing = true }, dir)) { - using (DBreeze.Transactions.Transaction transaction = engine.GetTransaction()) + using (var batch = new WriteBatch()) { ChainedHeader toSave = tip; var blocks = new List(); @@ -68,10 +71,10 @@ public void GetChainReturnsConcurrentChainFromDisk() foreach (ChainedHeader block in blocks) { - transaction.Insert("Chain", block.Height, this.dBreezeSerializer.Serialize(block.Header)); + batch.Put(BitConverter.GetBytes(block.Height), this.dBreezeSerializer.Serialize(block.Header)); } - transaction.Commit(); + engine.Write(batch); } } using (var repo = new ChainRepository(dir, new LoggerFactory(), this.dBreezeSerializer, new MemoryHeaderStore())) diff --git a/src/Blockcore.Tests/FinalizedBlockInfoRepositoryTest.cs b/src/Blockcore.Tests/FinalizedBlockInfoRepositoryTest.cs index 26fda8011..d6bfebff4 100644 --- a/src/Blockcore.Tests/FinalizedBlockInfoRepositoryTest.cs +++ b/src/Blockcore.Tests/FinalizedBlockInfoRepositoryTest.cs @@ -1,5 +1,6 @@ using System.Threading.Tasks; using Blockcore.AsyncWork; +using Blockcore.Consensus; using Blockcore.Tests.Common; using Blockcore.Utilities; using Microsoft.Extensions.Logging; diff --git a/src/Blockcore/Base/ChainRepository.cs b/src/Blockcore/Base/ChainRepository.cs index b04402360..bd59a81b1 100644 --- a/src/Blockcore/Base/ChainRepository.cs +++ b/src/Blockcore/Base/ChainRepository.cs @@ -5,8 +5,7 @@ using System.Threading.Tasks; using Blockcore.Configuration; using Blockcore.Utilities; -using DBreeze; -using DBreeze.DataTypes; +using LevelDB; using Microsoft.Extensions.Logging; using NBitcoin; @@ -30,8 +29,8 @@ public class ChainRepository : IChainRepository /// Instance logger. private readonly ILogger logger; - /// Access to DBreeze database. - private readonly DBreezeEngine dbreeze; + /// Access to database. + private readonly DB leveldb; private BlockLocator locator; @@ -45,7 +44,10 @@ public ChainRepository(string folder, ILoggerFactory loggerFactory, DBreezeSeria this.logger = loggerFactory.CreateLogger(this.GetType().FullName); Directory.CreateDirectory(folder); - this.dbreeze = new DBreezeEngine(folder); + + // Open a connection to a new DB and create if not found + var options = new Options { CreateIfMissing = true }; + this.leveldb = new DB(options, folder); } public ChainRepository(DataFolder dataFolder, ILoggerFactory loggerFactory, DBreezeSerializer dBreezeSerializer, IBlockHeaderStore blockHeaderStore) @@ -58,44 +60,48 @@ public Task LoadAsync(ChainedHeader genesisHeader) { Task task = Task.Run(() => { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) + ChainedHeader tip = null; + + byte[] firstRow = this.leveldb.Get(BitConverter.GetBytes(0)); + + if (firstRow == null) { - transaction.ValuesLazyLoadingIsOn = false; - ChainedHeader tip = null; - Row firstRow = transaction.Select("Chain", 0); + genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); + return genesisHeader; + } - if (!firstRow.Exists) - { - genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); - return genesisHeader; - } + BlockHeader nextHeader = this.dBreezeSerializer.Deserialize(firstRow); + Guard.Assert(nextHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks - BlockHeader nextHeader = this.dBreezeSerializer.Deserialize(firstRow.Value); - Guard.Assert(nextHeader.GetHash() == genesisHeader.HashBlock); // can't swap networks + int index = 1; + while (true) + { + byte[] row = this.leveldb.Get(BitConverter.GetBytes(index)); - foreach (Row row in transaction.SelectForwardSkip("Chain", 1)) - { - if ((tip != null) && (nextHeader.HashPrevBlock != tip.HashBlock)) - break; + if (row == null) + break; - BlockHeader blockHeader = this.dBreezeSerializer.Deserialize(row.Value); - tip = new ChainedHeader(nextHeader, blockHeader.HashPrevBlock, tip); - if (tip.Height == 0) tip.SetBlockHeaderStore(this.blockHeaderStore); - nextHeader = blockHeader; - } + if ((tip != null) && (nextHeader.HashPrevBlock != tip.HashBlock)) + break; - if (nextHeader != null) - tip = new ChainedHeader(nextHeader, nextHeader.GetHash(), tip); + BlockHeader blockHeader = this.dBreezeSerializer.Deserialize(row); + tip = new ChainedHeader(nextHeader, blockHeader.HashPrevBlock, tip); + if (tip.Height == 0) tip.SetBlockHeaderStore(this.blockHeaderStore); + nextHeader = blockHeader; + index++; + } - if (tip == null) - { - genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); - tip = genesisHeader; - } + if (nextHeader != null) + tip = new ChainedHeader(nextHeader, nextHeader.GetHash(), tip); - this.locator = tip.GetLocator(); - return tip; + if (tip == null) + { + genesisHeader.SetBlockHeaderStore(this.blockHeaderStore); + tip = genesisHeader; } + + this.locator = tip.GetLocator(); + return tip; }); return task; @@ -108,7 +114,7 @@ public Task SaveAsync(ChainIndexer chainIndexer) Task task = Task.Run(() => { - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) + using (var batch = new WriteBatch()) { ChainedHeader fork = this.locator == null ? null : chainIndexer.FindFork(this.locator); ChainedHeader tip = chainIndexer.Tip; @@ -140,11 +146,11 @@ public Task SaveAsync(ChainIndexer chainIndexer) header = newHeader; } - transaction.Insert("Chain", block.Height, this.dBreezeSerializer.Serialize(header)); + batch.Put(BitConverter.GetBytes(block.Height), this.dBreezeSerializer.Serialize(header)); } this.locator = tip.GetLocator(); - transaction.Commit(); + this.leveldb.Write(batch); } }); @@ -154,7 +160,8 @@ public Task SaveAsync(ChainIndexer chainIndexer) /// public void Dispose() { - this.dbreeze?.Dispose(); + this.leveldb?.Dispose(); + (this.blockHeaderStore as IDisposable)?.Dispose(); } } } \ No newline at end of file diff --git a/src/Blockcore/Consensus/ConsensusManager.cs b/src/Blockcore/Consensus/ConsensusManager.cs index 1bb9c3f8c..145c9405d 100644 --- a/src/Blockcore/Consensus/ConsensusManager.cs +++ b/src/Blockcore/Consensus/ConsensusManager.cs @@ -39,6 +39,9 @@ public class ConsensusManager : IConsensusManager /// The maximum amount of blocks that can be assigned to at the same time. private const int MaxBlocksToAskFromPuller = 10000; + /// The maximum amount of blocks that the can store. + private const int MaxUnconsumedBlocksCount = 10000; + /// The minimum amount of slots that should be available to trigger asking block puller for blocks. private const int ConsumptionThresholdSlots = MaxBlocksToAskFromPuller / 10; @@ -1322,6 +1325,12 @@ private void ProcessDownloadQueueLocked() return; } + if (this.chainedHeaderTree.UnconsumedBlocksCount > MaxUnconsumedBlocksCount) + { + this.logger.LogTrace("(-)[MAX_UNCONSUMED_BLOCKS_REACHED]"); + return; + } + long freeBytes = this.maxUnconsumedBlocksDataBytes - this.chainedHeaderTree.UnconsumedBlocksDataBytes - this.expectedBlockDataBytes; this.logger.LogDebug("{0} bytes worth of blocks is available for download.", freeBytes); diff --git a/src/Blockcore/FinalizedBlockInfoRepository.cs b/src/Blockcore/Consensus/FinalizedBlockInfoRepository.cs similarity index 99% rename from src/Blockcore/FinalizedBlockInfoRepository.cs rename to src/Blockcore/Consensus/FinalizedBlockInfoRepository.cs index f3dca7802..7022cfbd6 100644 --- a/src/Blockcore/FinalizedBlockInfoRepository.cs +++ b/src/Blockcore/Consensus/FinalizedBlockInfoRepository.cs @@ -7,7 +7,7 @@ using Microsoft.Extensions.Logging; using NBitcoin; -namespace Blockcore +namespace Blockcore.Consensus { /// Provider of the last finalized block's height and hash. /// diff --git a/src/Blockcore/Consensus/LeveldbHeaderStore.cs b/src/Blockcore/Consensus/LeveldbHeaderStore.cs index b9f643b03..3c49c1311 100644 --- a/src/Blockcore/Consensus/LeveldbHeaderStore.cs +++ b/src/Blockcore/Consensus/LeveldbHeaderStore.cs @@ -5,7 +5,7 @@ namespace NBitcoin { - public class LeveldbHeaderStore : IBlockHeaderStore + public class LeveldbHeaderStore : IBlockHeaderStore, IDisposable { private readonly Network network; @@ -87,5 +87,10 @@ public bool StoreHeader(BlockHeader blockHeader) return true; } + + public void Dispose() + { + this.leveldb?.Dispose(); + } } } \ No newline at end of file diff --git a/src/Blockcore/Utilities/KeyValueRepository.cs b/src/Blockcore/Utilities/KeyValueRepository.cs index 7c6fc5815..bafe96b24 100644 --- a/src/Blockcore/Utilities/KeyValueRepository.cs +++ b/src/Blockcore/Utilities/KeyValueRepository.cs @@ -3,8 +3,7 @@ using System.Text; using Blockcore.Configuration; using Blockcore.Utilities.JsonConverters; -using DBreeze; -using DBreeze.DataTypes; +using LevelDB; namespace Blockcore.Utilities { @@ -32,22 +31,23 @@ public interface IKeyValueRepository : IDisposable public class KeyValueRepository : IKeyValueRepository { - /// Access to DBreeze database. - private readonly DBreezeEngine dbreeze; - - private const string TableName = "common"; + /// Access to database. + private readonly DB leveldb; private readonly DBreezeSerializer dBreezeSerializer; - public KeyValueRepository(DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this (dataFolder.KeyValueRepositoryPath, dBreezeSerializer) + public KeyValueRepository(DataFolder dataFolder, DBreezeSerializer dBreezeSerializer) : this(dataFolder.KeyValueRepositoryPath, dBreezeSerializer) { } public KeyValueRepository(string folder, DBreezeSerializer dBreezeSerializer) { Directory.CreateDirectory(folder); - this.dbreeze = new DBreezeEngine(folder); this.dBreezeSerializer = dBreezeSerializer; + + // Open a connection to a new DB and create if not found + var options = new Options { CreateIfMissing = true }; + this.leveldb = new DB(options, folder); } /// @@ -55,12 +55,7 @@ public void SaveBytes(string key, byte[] bytes) { byte[] keyBytes = Encoding.ASCII.GetBytes(key); - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) - { - transaction.Insert(TableName, keyBytes, bytes); - - transaction.Commit(); - } + this.leveldb.Put(keyBytes, bytes); } /// @@ -83,17 +78,12 @@ public byte[] LoadBytes(string key) { byte[] keyBytes = Encoding.ASCII.GetBytes(key); - using (DBreeze.Transactions.Transaction transaction = this.dbreeze.GetTransaction()) - { - transaction.ValuesLazyLoadingIsOn = false; - - Row row = transaction.Select(TableName, keyBytes); + byte[] row = this.leveldb.Get(keyBytes); - if (!row.Exists) - return null; + if (row == null) + return null; - return row.Value; - } + return row; } /// @@ -126,7 +116,7 @@ public T LoadValueJson(string key) /// public void Dispose() { - this.dbreeze.Dispose(); + this.leveldb.Dispose(); } } -} +} \ No newline at end of file diff --git a/src/Blockcore/Utilities/LeveldbHelper.cs b/src/Blockcore/Utilities/LeveldbHelper.cs new file mode 100644 index 000000000..cbfecdeec --- /dev/null +++ b/src/Blockcore/Utilities/LeveldbHelper.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using LevelDB; + +namespace Blockcore.Utilities +{ + public static class DBH + { + public static byte[] Key(byte table, byte[] key) + { + Span dbkey = stackalloc byte[key.Length + 1]; + dbkey[0] = table; + key.AsSpan().CopyTo(dbkey.Slice(1)); + return dbkey.ToArray(); + } + + public static Dictionary SelectDictionary(this DB db, byte table) + { + var dict = new Dictionary(); + + var enumerator = db.GetEnumerator(); + while (enumerator.MoveNext()) + { + if (enumerator.Current.Key[0] == table) + dict.Add(enumerator.Current.Key.AsSpan().Slice(1).ToArray(), enumerator.Current.Value); + } + + return dict; + } + } +} \ No newline at end of file