Skip to content

Commit 5012a0a

Browse files
dangershonydangershony
committed
Fix prunning
1 parent 366e55b commit 5012a0a

9 files changed

Lines changed: 85 additions & 248 deletions

File tree

‎src/Features/Blockcore.Features.BlockStore/BlockRepository.cs‎

Lines changed: 23 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ public interface IBlockRepository : IBlockStore
2222
/// <summary> The dbreeze database engine.</summary>
2323
DB Leveldb { get; }
2424

25+
/// <summary>Really ugly temporary hack.</summary>
26+
object Locker { get; }
27+
2528
/// <summary>
2629
/// Deletes blocks and indexes for transactions that belong to deleted blocks.
2730
/// <para>
@@ -82,7 +85,7 @@ public class BlockRepository : IBlockRepository
8285

8386
private readonly DB leveldb;
8487

85-
private object locker;
88+
public object Locker { get; }
8689

8790
private readonly ILogger logger;
8891

@@ -117,7 +120,7 @@ public BlockRepository(Network network, string folder, ILoggerFactory loggerFact
117120
Directory.CreateDirectory(folder);
118121
var options = new Options { CreateIfMissing = true };
119122
this.leveldb = new DB(options, folder);
120-
this.locker = new object();
123+
this.Locker = new object();
121124

122125
this.logger = loggerFactory.CreateLogger(this.GetType().FullName);
123126
this.network = network;
@@ -130,7 +133,7 @@ public virtual void Initialize()
130133
{
131134
Block genesis = this.network.GetGenesis();
132135

133-
lock (this.locker)
136+
lock (this.Locker)
134137
{
135138
if (this.LoadTipHashAndHeight() == null)
136139
{
@@ -161,7 +164,7 @@ public Transaction GetTransactionById(uint256 trxid)
161164
}
162165

163166
Transaction res = null;
164-
lock (this.locker)
167+
lock (this.Locker)
165168
{
166169
byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxid.ToBytes()));
167170

@@ -194,7 +197,7 @@ public Transaction GetTransactionById(uint256 trxid)
194197

195198
Transaction[] txes = new Transaction[trxids.Length];
196199

197-
lock (this.locker)
200+
lock (this.Locker)
198201
{
199202
for (int i = 0; i < trxids.Length; i++)
200203
{
@@ -258,7 +261,7 @@ public uint256 GetBlockIdByTransactionId(uint256 trxid)
258261
}
259262

260263
uint256 res = null;
261-
lock (this.locker)
264+
lock (this.Locker)
262265
{
263266
byte[] transactionRow = this.leveldb.Get(DBH.Key(TransactionTableName, trxid.ToBytes()));
264267
if (transactionRow != null)
@@ -293,17 +296,12 @@ protected virtual void OnInsertBlocks(List<Block> blocks)
293296
uint256 blockId = kv.Key;
294297
Block block = kv.Value;
295298

296-
// If the block is already in store don't write it again.
297-
byte[] blockRow = this.leveldb.Get(DBH.Key(BlockTableName, blockId.ToBytes()));
298-
if (blockRow == null)
299-
{
300-
batch.Put(DBH.Key(BlockTableName, blockId.ToBytes()), this.dataStoreSerializer.Serialize(block));
299+
batch.Put(DBH.Key(BlockTableName, blockId.ToBytes()), this.dataStoreSerializer.Serialize(block));
301300

302-
if (this.TxIndex)
303-
{
304-
foreach (Transaction transaction in block.Transactions)
305-
transactions.Add((transaction, block));
306-
}
301+
if (this.TxIndex)
302+
{
303+
foreach (Transaction transaction in block.Transactions)
304+
transactions.Add((transaction, block));
307305
}
308306
}
309307

@@ -331,7 +329,7 @@ protected virtual void OnInsertTransactions(List<(Transaction, Block)> transacti
331329

332330
public IEnumerable<Block> EnumeratehBatch(List<ChainedHeader> headers)
333331
{
334-
lock (this.locker)
332+
lock (this.Locker)
335333
{
336334
foreach (ChainedHeader chainedHeader in headers)
337335
{
@@ -345,7 +343,7 @@ public IEnumerable<Block> EnumeratehBatch(List<ChainedHeader> headers)
345343
/// <inheritdoc />
346344
public void ReIndex()
347345
{
348-
lock (this.locker)
346+
lock (this.Locker)
349347
{
350348
if (this.TxIndex)
351349
{
@@ -411,7 +409,7 @@ public void PutBlocks(HashHeightPair newTip, List<Block> blocks)
411409

412410
// DBreeze is faster if sort ascending by key in memory before insert
413411
// however we need to find how byte arrays are sorted in DBreeze.
414-
lock (this.locker)
412+
lock (this.Locker)
415413
{
416414
this.OnInsertBlocks(blocks);
417415

@@ -442,7 +440,7 @@ private void SaveTxIndex(bool txIndex)
442440
/// <inheritdoc />
443441
public void SetTxIndex(bool txIndex)
444442
{
445-
lock (this.locker)
443+
lock (this.Locker)
446444
{
447445
this.SaveTxIndex(txIndex);
448446
}
@@ -472,7 +470,7 @@ public Block GetBlock(uint256 hash)
472470
Guard.NotNull(hash, nameof(hash));
473471

474472
Block res = null;
475-
lock (this.locker)
473+
lock (this.Locker)
476474
{
477475
var results = this.GetBlocksFromHashes(new List<uint256> { hash });
478476

@@ -490,7 +488,7 @@ public List<Block> GetBlocks(List<uint256> hashes)
490488

491489
List<Block> blocks;
492490

493-
lock (this.locker)
491+
lock (this.Locker)
494492
{
495493
blocks = this.GetBlocksFromHashes(hashes);
496494
}
@@ -504,7 +502,7 @@ public bool Exist(uint256 hash)
504502
Guard.NotNull(hash, nameof(hash));
505503

506504
bool res = false;
507-
lock (this.locker)
505+
lock (this.Locker)
508506
{
509507
// Lazy loading is on so we don't fetch the whole value, just the row.
510508
byte[] key = hash.ToBytes();
@@ -583,7 +581,7 @@ public void Delete(HashHeightPair newTip, List<uint256> hashes)
583581
Guard.NotNull(newTip, nameof(newTip));
584582
Guard.NotNull(hashes, nameof(hashes));
585583

586-
lock (this.locker)
584+
lock (this.Locker)
587585
{
588586
List<Block> blocks = this.GetBlocksFromHashes(hashes);
589587
this.OnDeleteBlocks(blocks.Where(b => b != null).ToList());
@@ -596,7 +594,7 @@ public void DeleteBlocks(List<uint256> hashes)
596594
{
597595
Guard.NotNull(hashes, nameof(hashes));
598596

599-
lock (this.locker)
597+
lock (this.Locker)
600598
{
601599
List<Block> blocks = this.GetBlocksFromHashes(hashes);
602600

‎src/Features/Blockcore.Features.BlockStore/BlockStoreFeature.cs‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ public class BlockStoreFeature : FullNodeFeature
4848
private readonly IPrunedBlockRepository prunedBlockRepository;
4949

5050
private readonly IAddressIndexer addressIndexer;
51+
private readonly IPruneBlockStoreService pruneBlockStoreService;
5152

5253
public BlockStoreFeature(
5354
Network network,
@@ -62,7 +63,8 @@ public BlockStoreFeature(
6263
IConsensusManager consensusManager,
6364
ICheckpoints checkpoints,
6465
IPrunedBlockRepository prunedBlockRepository,
65-
IAddressIndexer addressIndexer)
66+
IAddressIndexer addressIndexer,
67+
IPruneBlockStoreService pruneBlockStoreService)
6668
{
6769
this.network = network;
6870
this.chainIndexer = chainIndexer;
@@ -77,7 +79,7 @@ public BlockStoreFeature(
7779
this.checkpoints = checkpoints;
7880
this.prunedBlockRepository = prunedBlockRepository;
7981
this.addressIndexer = addressIndexer;
80-
82+
this.pruneBlockStoreService = pruneBlockStoreService;
8183
nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, this.GetType().Name, 900);
8284
}
8385

@@ -91,6 +93,15 @@ private void AddInlineStats(StringBuilder log)
9193
builder.Append("BlockStore.Height: ".PadRight(LoggingConfiguration.ColumnLength + 1) + highestBlock.Height.ToString().PadRight(8));
9294
builder.Append(" BlockStore.Hash: ".PadRight(LoggingConfiguration.ColumnLength - 1) + highestBlock.HashBlock);
9395
log.AppendLine(builder.ToString());
96+
97+
if (this.storeSettings.PruningEnabled)
98+
{
99+
builder = new StringBuilder();
100+
var prunedTip = this.prunedBlockRepository.PrunedTip;
101+
builder.Append("PrunedStore.Height: ".PadRight(LoggingConfiguration.ColumnLength + 1) + prunedTip.Height.ToString().PadRight(8));
102+
builder.Append(" PrunedStore.Hash: ".PadRight(LoggingConfiguration.ColumnLength - 1) + prunedTip.Hash);
103+
log.AppendLine(builder.ToString());
104+
}
94105
}
95106
}
96107

@@ -125,8 +136,10 @@ public override Task InitializeAsync()
125136
if (this.storeSettings.AmountOfBlocksToKeep < this.network.Consensus.MaxReorgLength)
126137
throw new BlockStoreException($"The amount of blocks to prune [{this.storeSettings.AmountOfBlocksToKeep}] (blocks to keep) cannot be less than the node's max reorg length of {this.network.Consensus.MaxReorgLength}.");
127138

128-
this.logger.LogInformation("Pruning BlockStore...");
129-
this.prunedBlockRepository.PruneAndCompactDatabase(this.chainState.BlockStoreTip, this.network, true);
139+
this.prunedBlockRepository.PrepareDatabase();
140+
141+
this.logger.LogInformation("Starting Prunning...");
142+
this.pruneBlockStoreService.Initialize();
130143
}
131144

132145
// Use ProvenHeadersBlockStoreBehavior for PoS Networks
@@ -155,8 +168,8 @@ public override void Dispose()
155168
{
156169
if (this.storeSettings.PruningEnabled)
157170
{
158-
this.logger.LogInformation("Pruning BlockStore...");
159-
this.prunedBlockRepository.PruneAndCompactDatabase(this.chainState.BlockStoreTip, this.network, false);
171+
this.logger.LogInformation("Stopping Prunning...");
172+
this.pruneBlockStoreService.Dispose();
160173
}
161174

162175
this.logger.LogInformation("Stopping BlockStoreSignaled.");
@@ -184,7 +197,6 @@ public static IFullNodeBuilder UseBlockStore(this IFullNodeBuilder fullNodeBuild
184197
{
185198
services.AddSingleton<IBlockStoreQueue, BlockStoreQueue>().AddSingleton<IBlockStore>(provider => provider.GetService<IBlockStoreQueue>());
186199
services.AddSingleton<IBlockRepository, BlockRepository>();
187-
services.AddSingleton<IPrunedBlockRepository, PrunedBlockRepository>();
188200

189201
if (fullNodeBuilder.Network.Consensus.IsProofOfStake)
190202
services.AddSingleton<BlockStoreSignaled, ProvenHeadersBlockStoreSignaled>();
@@ -194,6 +206,9 @@ public static IFullNodeBuilder UseBlockStore(this IFullNodeBuilder fullNodeBuild
194206
services.AddSingleton<StoreSettings>();
195207
services.AddSingleton<IBlockStoreQueueFlushCondition, BlockStoreQueueFlushCondition>();
196208
services.AddSingleton<IAddressIndexer, AddressIndexer>();
209+
210+
services.AddSingleton<IPrunedBlockRepository, PrunedBlockRepository>();
211+
services.AddSingleton<IPruneBlockStoreService, PruneBlockStoreService>();
197212
});
198213
});
199214

‎src/Features/Blockcore.Features.BlockStore/Pruning/IPrunedBlockRepository.cs‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,6 @@ public interface IPrunedBlockRepository
1414
void Initialize();
1515

1616
/// <summary>
17-
/// Prunes and compacts the block and transaction database.
18-
/// <para>
19-
/// The method first prunes by deleting blocks from the block store that are below the <see cref="StoreSettings.AmountOfBlocksToKeep"/> from the store tip.
20-
/// </para>
21-
/// <para>
22-
/// Once this is done the database is compacted by resaving the database file without the deleted references, reducing the file size on disk.
23-
/// </para>
24-
/// </summary>
25-
/// <param name="blockStoreTip">The current tip of the store.</param>
26-
/// <param name="network">The network the node is running on.</param>
27-
/// <param name="nodeInitializing">Indicates whether or not this method is called from node startup or not.</param>
28-
void PruneAndCompactDatabase(ChainedHeader blockStoreTip, Network network, bool nodeInitializing);
29-
30-
/// <summary>
3117
/// The lowest block hash and height that the repository has.
3218
/// <para>
3319
/// This also indicated where the node has been pruned up to.
@@ -37,11 +23,16 @@ public interface IPrunedBlockRepository
3723

3824
/// <summary>
3925
/// Sets the pruned tip.
40-
/// <para>
26+
/// <para>
4127
/// It will be saved once the block database has been compacted on node initialization or shutdown.
4228
/// </para>
4329
/// </summary>
4430
/// <param name="tip">The tip to set.</param>
4531
void UpdatePrunedTip(ChainedHeader tip);
32+
33+
/// <summary>
34+
/// Prepare the pruned tip.
35+
/// </summary>
36+
void PrepareDatabase();
4637
}
47-
}
38+
}

‎src/Features/Blockcore.Features.BlockStore/Pruning/PruneBlockStoreService.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,8 @@ public void Initialize()
5353
return Task.CompletedTask;
5454
},
5555
this.nodeLifetime.ApplicationStopping,
56-
repeatEvery: TimeSpans.TenSeconds);
56+
repeatEvery: TimeSpans.Minute,
57+
startAfter: TimeSpans.Minute);
5758
}
5859

5960
/// <inheritdoc/>
@@ -115,4 +116,4 @@ public void Dispose()
115116
this.asyncLoop?.Dispose();
116117
}
117118
}
118-
}
119+
}

0 commit comments

Comments
 (0)