From 26656098bee1869c752bf207ced6239442771fa4 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 25 Oct 2023 18:39:19 +0300 Subject: [PATCH 01/14] init --- .../Nethermind.Runner/configs/mainnet.cfg | 4 +-- .../FastBlocks/BodiesSyncFeed.cs | 15 ++++++--- .../FastBlocks/ReceiptsSyncFeed.cs | 33 ++++++++----------- 3 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg b/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg index 7f6fecd07c4..cd4f9177754 100644 --- a/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg +++ b/src/Nethermind/Nethermind.Runner/configs/mainnet.cfg @@ -16,8 +16,6 @@ "PivotHash": "0x621cd6a2635a8746f0889cfb81a63d24933dca62cd3dd1b7bcc87c73475b1923", "PivotTotalDifficulty": "58750003716598352816469", "FastBlocks": true, - "AncientBodiesBarrier": 11052984, - "AncientReceiptsBarrier": 11052984, "FastSyncCatchUpHeightDelta": "10000000000" }, "EthStats": { @@ -41,4 +39,4 @@ "Merge": { "Enabled": true } -} \ No newline at end of file +} diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index cb06445c567..dc93a9bec1f 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -21,6 +21,7 @@ namespace Nethermind.Synchronization.FastBlocks { public class BodiesSyncFeed : ActivatedSyncFeed { + private const int DepositContractBarrier = 11052984; private int _requestSize = GethSyncLimits.MaxBodyFetch; private const long DefaultFlushDbInterval = 100000; // About every 10GB on mainnet private readonly long _flushDbInterval; // About every 10GB on mainnet @@ -38,6 +39,14 @@ public class BodiesSyncFeed : ActivatedSyncFeed private SyncStatusList _syncStatusList; + private bool ShouldFinish => !_syncConfig.DownloadBodiesInFastSync || AllDownloaded; + private bool AllDownloaded => _blockTree.LowestInsertedBodyNumber <= _barrier + || WithinOldBarrierDefault; + + // This property was introduced when we switched defaults of barriers from 11052984 to 0 to not disturb existing node operators + private bool WithinOldBarrierDefault => _blockTree.LowestInsertedBodyNumber <= DepositContractBarrier + && _blockTree.LowestInsertedBodyNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; + public BodiesSyncFeed( IBlockTree blockTree, ISyncPeerPool syncPeerPool, @@ -94,10 +103,7 @@ private void ResetSyncStatusList() private bool ShouldBuildANewBatch() { - bool shouldDownloadBodies = _syncConfig.DownloadBodiesInFastSync; - bool allBodiesDownloaded = _syncStatusList.LowestInsertWithoutGaps <= _barrier; - bool shouldFinish = !shouldDownloadBodies || allBodiesDownloaded; - if (shouldFinish) + if (ShouldFinish) { ResetSyncStatusList(); Finish(); @@ -105,7 +111,6 @@ private bool ShouldBuildANewBatch() return false; } - return true; } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 2ad64024d00..1c650e3a424 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -23,7 +23,7 @@ namespace Nethermind.Synchronization.FastBlocks { public class ReceiptsSyncFeed : ActivatedSyncFeed { - private const int MinReceiptBlock = 1; + private const int DepositContractBarrier = 11052984; private int _requestSize = GethSyncLimits.MaxReceiptFetch; private readonly ILogger _logger; @@ -38,10 +38,15 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed private long _pivotNumber; private long _barrier; - private bool ShouldFinish => !_syncConfig.DownloadReceiptsInFastSync || AllReceiptsDownloaded; - private bool AllReceiptsDownloaded => _receiptStorage.LowestInsertedReceiptBlockNumber <= _barrier; - public override bool IsFinished => AllReceiptsDownloaded; + private bool ShouldFinish => !_syncConfig.DownloadReceiptsInFastSync || AllDownloaded; + private bool AllDownloaded => _receiptStorage.LowestInsertedReceiptBlockNumber <= _barrier + || WithinOldBarrierDefault; + + // This property was introduced when we switched defaults of barriers from 11052984 to 0 to not disturb existing node operators + private bool WithinOldBarrierDefault => _receiptStorage.LowestInsertedReceiptBlockNumber <= DepositContractBarrier + && _receiptStorage.LowestInsertedReceiptBlockNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; // this is intentional. using this as an approxamation assuming a minimum of 1 receipt in per block + public override bool IsFinished => AllReceiptsDownloaded; public ReceiptsSyncFeed( ISpecProvider specProvider, IBlockTree blockTree, @@ -98,25 +103,13 @@ private void ResetSyncStatusList() private bool ShouldBuildANewBatch() { - bool shouldDownloadReceipts = _syncConfig.DownloadReceiptsInFastSync; - bool allReceiptsDownloaded = AllReceiptsDownloaded; - bool isGenesisDownloaded = _syncStatusList.LowestInsertWithoutGaps <= MinReceiptBlock; - bool noBatchesLeft = !shouldDownloadReceipts - || allReceiptsDownloaded - || isGenesisDownloaded; - - if (noBatchesLeft) + if (ShouldFinish) { - if (ShouldFinish) - { - ResetSyncStatusList(); - Finish(); - PostFinishCleanUp(); - } - + ResetSyncStatusList(); + Finish(); + PostFinishCleanUp(); return false; } - return true; } From 545e0a706bf5300e16d1467f94c650e1cf3b1d3f Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 25 Oct 2023 19:20:01 +0300 Subject: [PATCH 02/14] tests --- .../FastSync/ReceiptsSyncFeedTests.cs | 14 ++++++++++++++ .../FastBlocks/ReceiptsSyncFeed.cs | 6 +++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs index cbeedc6625f..c83c47d1ecd 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs @@ -226,6 +226,20 @@ public async Task When_configured_to_skip_receipts_then_finishes_immediately() _measuredProgressQueue.HasEnded.Should().BeTrue(); } + [Test] + public async Task When_finished_sync_with_old_default_barrier_then_finishes_imedietely() + { + _syncConfig.AncientReceiptsBarrier = 0; + LoadScenario(_256BodiesWithOneTxEach); + _receiptStorage.LowestInsertedReceiptBlockNumber.Returns(ReceiptsSyncFeed.DepositContractBarrier - 50); + + ReceiptsSyncBatch? request = await _feed.PrepareRequest(); + request.Should().BeNull(); + _feed.CurrentState.Should().Be(SyncFeedState.Finished); + _measuredProgress.HasEnded.Should().BeTrue(); + _measuredProgressQueue.HasEnded.Should().BeTrue(); + } + private void LoadScenario(Scenario scenario) { LoadScenario(scenario, _syncConfig); diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 1c650e3a424..bc890877f7d 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -3,6 +3,7 @@ using System; using System.IO; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain; @@ -19,11 +20,14 @@ using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SyncLimits; +[assembly: InternalsVisibleTo("Nethermind.Synchronization.Test")] + namespace Nethermind.Synchronization.FastBlocks { + public class ReceiptsSyncFeed : ActivatedSyncFeed { - private const int DepositContractBarrier = 11052984; + internal const int DepositContractBarrier = 11052984; private int _requestSize = GethSyncLimits.MaxReceiptFetch; private readonly ILogger _logger; From a79085de8b807819fc889d9365dd677c06f109a9 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 25 Oct 2023 19:40:28 +0300 Subject: [PATCH 03/14] format --- .../Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index bc890877f7d..4500aa1eef2 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -24,7 +24,7 @@ namespace Nethermind.Synchronization.FastBlocks { - + public class ReceiptsSyncFeed : ActivatedSyncFeed { internal const int DepositContractBarrier = 11052984; From b0b89cccf98a254406a0c5a6a3467795036070f6 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 25 Oct 2023 19:58:53 +0300 Subject: [PATCH 04/14] fix config tests --- src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs b/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs index 6cb75b1de3e..9365d87cadf 100644 --- a/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs +++ b/src/Nethermind/Nethermind.Runner.Test/ConfigFilesTests.cs @@ -272,7 +272,7 @@ public void Migrations_are_not_enabled_by_default(string configWildcard) } [TestCase("^mainnet", 0)] - [TestCase("mainnet fast", 11052984)] + [TestCase("mainnet fast", 0)] public void Barriers_defaults_are_correct(string configWildcard, long barrier) { Test(configWildcard, c => c.AncientBodiesBarrier, barrier); From dfd01966707772872e5f4e33109d31487448287f Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 25 Oct 2023 20:28:03 +0300 Subject: [PATCH 05/14] cover more cases --- .../FastSync/ReceiptsSyncFeedTests.cs | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs index c83c47d1ecd..b0b87d196af 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs @@ -9,6 +9,7 @@ using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; +using Nethermind.Consensus; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Core.Specs; @@ -226,18 +227,30 @@ public async Task When_configured_to_skip_receipts_then_finishes_immediately() _measuredProgressQueue.HasEnded.Should().BeTrue(); } - [Test] - public async Task When_finished_sync_with_old_default_barrier_then_finishes_imedietely() + [TestCase(100, false)] + [TestCase(11052930, true)] + [TestCase(11052984, true)] + [TestCase(11052985, false)] + public async Task When_finished_sync_with_old_default_barrier_then_finishes_imedietely( + long? lowestInsertedReceiptBlockNumber, bool Shouldfinish) { _syncConfig.AncientReceiptsBarrier = 0; LoadScenario(_256BodiesWithOneTxEach); - _receiptStorage.LowestInsertedReceiptBlockNumber.Returns(ReceiptsSyncFeed.DepositContractBarrier - 50); + _receiptStorage.LowestInsertedReceiptBlockNumber.Returns(lowestInsertedReceiptBlockNumber); ReceiptsSyncBatch? request = await _feed.PrepareRequest(); - request.Should().BeNull(); - _feed.CurrentState.Should().Be(SyncFeedState.Finished); - _measuredProgress.HasEnded.Should().BeTrue(); - _measuredProgressQueue.HasEnded.Should().BeTrue(); + if (Shouldfinish) + { + request.Should().BeNull(); + _feed.CurrentState.Should().Be(SyncFeedState.Finished); + } + else + { + request.Should().NotBeNull(); + _feed.CurrentState.Should().NotBe(SyncFeedState.Finished); + } + _measuredProgress.HasEnded.Should().Be(Shouldfinish); + _measuredProgressQueue.HasEnded.Should().Be(Shouldfinish); } private void LoadScenario(Scenario scenario) From 43b4a8d1f6c824059139f5076d9b94b14bcc0b27 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Thu, 26 Oct 2023 17:19:38 +0300 Subject: [PATCH 06/14] Add same logic to SyncProgressResolver with tests --- .../FastSync/ReceiptsSyncFeedTests.cs | 8 +++--- .../SyncProgressResolverTests.cs | 28 +++++++++++++++++++ .../FastBlocks/BodiesSyncFeed.cs | 2 +- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs index b0b87d196af..f3d38201f7e 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs @@ -232,14 +232,14 @@ public async Task When_configured_to_skip_receipts_then_finishes_immediately() [TestCase(11052984, true)] [TestCase(11052985, false)] public async Task When_finished_sync_with_old_default_barrier_then_finishes_imedietely( - long? lowestInsertedReceiptBlockNumber, bool Shouldfinish) + long? lowestInsertedReceiptBlockNumber, bool shouldfinish) { _syncConfig.AncientReceiptsBarrier = 0; LoadScenario(_256BodiesWithOneTxEach); _receiptStorage.LowestInsertedReceiptBlockNumber.Returns(lowestInsertedReceiptBlockNumber); ReceiptsSyncBatch? request = await _feed.PrepareRequest(); - if (Shouldfinish) + if (shouldfinish) { request.Should().BeNull(); _feed.CurrentState.Should().Be(SyncFeedState.Finished); @@ -249,8 +249,8 @@ public async Task When_configured_to_skip_receipts_then_finishes_immediately() request.Should().NotBeNull(); _feed.CurrentState.Should().NotBe(SyncFeedState.Finished); } - _measuredProgress.HasEnded.Should().Be(Shouldfinish); - _measuredProgressQueue.HasEnded.Should().Be(Shouldfinish); + _measuredProgress.HasEnded.Should().Be(shouldfinish); + _measuredProgressQueue.HasEnded.Should().Be(shouldfinish); } private void LoadScenario(Scenario scenario) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs index 3fa95df3df1..65442fcfd91 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs @@ -151,6 +151,34 @@ public void Is_fast_block_bodies_finished_returns_false_when_blocks_not_download Assert.False(syncProgressResolver.IsFastBlocksBodiesFinished()); } + [TestCase(100, false)] + [TestCase(11052930, true)] + [TestCase(11052984, true)] + [TestCase(11052985, false)] + public void Is_fast_block_bodies_and_receipts_finished_returns_correct_value_when_within_deposit_contract_barrier(long? lowestInsertedReceiptBlockNumber, bool shouldfinish) + { + IBlockTree blockTree = Substitute.For(); + IReceiptStorage receiptStorage = Substitute.For(); + IDb stateDb = new MemDb(); + SyncConfig syncConfig = new() + { + FastBlocks = true, + FastSync = true, + DownloadBodiesInFastSync = true, + DownloadReceiptsInFastSync = true, + PivotNumber = "1", + }; + ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); + + blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); + blockTree.LowestInsertedBodyNumber.Returns(lowestInsertedReceiptBlockNumber); + receiptStorage.LowestInsertedReceiptBlockNumber.Returns(lowestInsertedReceiptBlockNumber); + + SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, stateDb, NullTrieNodeResolver.Instance, progressTracker, syncConfig, LimboLogs.Instance); + Assert.That(shouldfinish, Is.EqualTo(syncProgressResolver.IsFastBlocksBodiesFinished())); + Assert.That(shouldfinish, Is.EqualTo(syncProgressResolver.IsFastBlocksReceiptsFinished())); + } + [Test] public void Is_fast_block_receipts_finished_returns_true_when_receipts_not_downloaded_and_we_do_not_want_to_download_receipts() { diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index dc93a9bec1f..7312b6c712f 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -21,7 +21,7 @@ namespace Nethermind.Synchronization.FastBlocks { public class BodiesSyncFeed : ActivatedSyncFeed { - private const int DepositContractBarrier = 11052984; + internal const int DepositContractBarrier = 11052984; private int _requestSize = GethSyncLimits.MaxBodyFetch; private const long DefaultFlushDbInterval = 100000; // About every 10GB on mainnet private readonly long _flushDbInterval; // About every 10GB on mainnet From ed1029a34e204b3ca287436b79d8a59394f47481 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Sat, 28 Oct 2023 23:06:02 +0300 Subject: [PATCH 07/14] wip metadatadb usage to detect with what configuration was the sync started. --- .../Nethermind.Db/MetadataDbKeys.cs | 2 ++ .../FastBlocks/ReceiptsSyncFeed.cs | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/Nethermind/Nethermind.Db/MetadataDbKeys.cs b/src/Nethermind/Nethermind.Db/MetadataDbKeys.cs index e6897a7d9a4..ff44b35b185 100644 --- a/src/Nethermind/Nethermind.Db/MetadataDbKeys.cs +++ b/src/Nethermind/Nethermind.Db/MetadataDbKeys.cs @@ -14,5 +14,7 @@ public static class MetadataDbKeys public const int LowestInsertedBeaconHeaderHash = 7; public const int FirstPoSHash = 8; public const int UpdatedPivotData = 9; + public const int ReceiptsBarrierWhenStarted = 10; + public const int BodiesBarrierWhenStarted = 11; } } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 4500aa1eef2..99c85f61d94 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -13,7 +13,9 @@ using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; using Nethermind.Core.Specs; +using Nethermind.Db; using Nethermind.Logging; +using Nethermind.Specs; using Nethermind.Stats.Model; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -37,10 +39,12 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed private readonly ISpecProvider _specProvider; private readonly IReceiptStorage _receiptStorage; private readonly ISyncPeerPool _syncPeerPool; + private readonly IDb _metadataDb; private SyncStatusList _syncStatusList; private long _pivotNumber; private long _barrier; + private readonly long? _barrierWhenStarted; private bool ShouldFinish => !_syncConfig.DownloadReceiptsInFastSync || AllDownloaded; private bool AllDownloaded => _receiptStorage.LowestInsertedReceiptBlockNumber <= _barrier @@ -58,6 +62,7 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, + IDb metadataDb, ILogManager logManager) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); @@ -67,6 +72,7 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); _syncReport = syncReport ?? throw new ArgumentNullException(nameof(syncReport)); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); + _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb)); if (!_syncConfig.FastBlocks) { @@ -84,6 +90,21 @@ public override void InitializeFeed() _barrier = _syncConfig.AncientReceiptsBarrierCalc; if (_logger.IsInfo) _logger.Info($"Changed pivot in receipts sync. Now using pivot {_pivotNumber} and barrier {_barrier}"); ResetSyncStatusList(); + if (specProvider.ChainId != BlockchainIds.Mainnet) + return; + + if (_receiptStorage.LowestInsertedReceiptBlockNumber is null) + { + if (!_metadataDb.KeyExists(MetadataDbKeys.ReceiptsBarrierWhenStarted)) + { + _barrierWhenStarted = _syncConfig.AncientReceiptsBarrier; + _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + } + if (_metadataDb.KeyExists(MetadataDbKeys.ReceiptsBarrierWhenStarted)) + { + _barrierWhenStarted = _metadataDb.Get(MetadataDbKeys.ReceiptsBarrierWhenStarted).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); + } } base.InitializeFeed(); From 3ad31deb2c06fc524b4b41dbc0c762f95799ff21 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Fri, 3 Nov 2023 16:00:54 +0300 Subject: [PATCH 08/14] add logic to ReceiptsSyncFeed --- .../FastBlocks/ReceiptsSyncFeed.cs | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 99c85f61d94..4b380735eb0 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -15,7 +15,6 @@ using Nethermind.Core.Specs; using Nethermind.Db; using Nethermind.Logging; -using Nethermind.Specs; using Nethermind.Stats.Model; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -50,8 +49,9 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed private bool AllDownloaded => _receiptStorage.LowestInsertedReceiptBlockNumber <= _barrier || WithinOldBarrierDefault; - // This property was introduced when we switched defaults of barriers from 11052984 to 0 to not disturb existing node operators - private bool WithinOldBarrierDefault => _receiptStorage.LowestInsertedReceiptBlockNumber <= DepositContractBarrier + // This property was introduced when we switched defaults of barriers on mainnet from 11052984 to 0 to not disturb existing node operators + private bool WithinOldBarrierDefault => _barrierWhenStarted == DepositContractBarrier + && _receiptStorage.LowestInsertedReceiptBlockNumber <= DepositContractBarrier && _receiptStorage.LowestInsertedReceiptBlockNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; // this is intentional. using this as an approxamation assuming a minimum of 1 receipt in per block public override bool IsFinished => AllReceiptsDownloaded; @@ -92,19 +92,20 @@ public override void InitializeFeed() ResetSyncStatusList(); if (specProvider.ChainId != BlockchainIds.Mainnet) return; - - if (_receiptStorage.LowestInsertedReceiptBlockNumber is null) + if (!_receiptStorage.HasBlock(_syncConfig.PivotNumberParsed, _syncConfig.PivotHashParsed)) { - if (!_metadataDb.KeyExists(MetadataDbKeys.ReceiptsBarrierWhenStarted)) - { - _barrierWhenStarted = _syncConfig.AncientReceiptsBarrier; - _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } + _barrierWhenStarted = _syncConfig.AncientReceiptsBarrierCalc; + _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); } - if (_metadataDb.KeyExists(MetadataDbKeys.ReceiptsBarrierWhenStarted)) + else if (_metadataDb.KeyExists(MetadataDbKeys.ReceiptsBarrierWhenStarted)) { _barrierWhenStarted = _metadataDb.Get(MetadataDbKeys.ReceiptsBarrierWhenStarted).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); } + else + { + // Assume the receipts barrier was the previous defualt + _barrierWhenStarted = DepositContractBarrier; + } } base.InitializeFeed(); From 8f8a7824a7b3157010e7d05a24fc6e548ebec881 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Fri, 3 Nov 2023 16:10:14 +0300 Subject: [PATCH 09/14] fix ReceiptsSyncFeed --- .../FastBlocks/ReceiptsSyncFeed.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 4b380735eb0..43162079004 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -43,7 +43,7 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed private SyncStatusList _syncStatusList; private long _pivotNumber; private long _barrier; - private readonly long? _barrierWhenStarted; + private long? _barrierWhenStarted; private bool ShouldFinish => !_syncConfig.DownloadReceiptsInFastSync || AllDownloaded; private bool AllDownloaded => _receiptStorage.LowestInsertedReceiptBlockNumber <= _barrier @@ -54,7 +54,7 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed && _receiptStorage.LowestInsertedReceiptBlockNumber <= DepositContractBarrier && _receiptStorage.LowestInsertedReceiptBlockNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; // this is intentional. using this as an approxamation assuming a minimum of 1 receipt in per block - public override bool IsFinished => AllReceiptsDownloaded; + public override bool IsFinished => AllDownloaded; public ReceiptsSyncFeed( ISpecProvider specProvider, IBlockTree blockTree, @@ -90,7 +90,7 @@ public override void InitializeFeed() _barrier = _syncConfig.AncientReceiptsBarrierCalc; if (_logger.IsInfo) _logger.Info($"Changed pivot in receipts sync. Now using pivot {_pivotNumber} and barrier {_barrier}"); ResetSyncStatusList(); - if (specProvider.ChainId != BlockchainIds.Mainnet) + if (_specProvider.ChainId != BlockchainIds.Mainnet) return; if (!_receiptStorage.HasBlock(_syncConfig.PivotNumberParsed, _syncConfig.PivotHashParsed)) { From 74cc0ea113dbf2a16380752372d2679e6c657508 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Fri, 3 Nov 2023 19:50:35 +0300 Subject: [PATCH 10/14] apply fix --- .../FastBlocks/BodiesSyncFeedTests.cs | 89 +++++++++++++++---- .../FastSync/ReceiptsSyncFeedTests.cs | 37 ++++++-- .../ReceiptSyncFeedTests.cs | 2 + .../SyncProgressResolverTests.cs | 29 +----- .../FastBlocks/BodiesSyncFeed.cs | 36 +++++++- .../FastBlocks/ReceiptsSyncFeed.cs | 15 ++-- .../Synchronizer.cs | 4 +- 7 files changed, 151 insertions(+), 61 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs index b082975a36b..324eb0c8ae6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs @@ -8,9 +8,12 @@ using Nethermind.Blockchain; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; +using Nethermind.Core.Extensions; using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; +using Nethermind.Db; using Nethermind.Logging; +using Nethermind.Specs; using Nethermind.Synchronization.FastBlocks; using Nethermind.Synchronization.ParallelSync; using Nethermind.Synchronization.Peers; @@ -22,9 +25,13 @@ namespace Nethermind.Synchronization.Test.FastBlocks; public class BodiesSyncFeedTests { - private BlockTree _syncingFromBlockTree = null!; + private IBlockTree _syncingFromBlockTree = null!; + private IBlockTree _syncingToBlockTree = null!; private TestMemDb _blocksDb = null!; - private BodiesSyncFeed _syncFeed = null!; + private BodiesSyncFeed _feed = null!; + private ISyncConfig _syncConfig = null!; + private MemDb _metadataDb = null!; + private Block _pivotBlock = null!; [SetUp] public void Setup() @@ -34,44 +41,47 @@ public void Setup() .TestObject; _blocksDb = new TestMemDb(); - BlockTree syncingTooBlockTree = Build.A.BlockTree() + _metadataDb = new MemDb(); + _syncingToBlockTree = Build.A.BlockTree() .WithBlocksDb(_blocksDb) .TestObject; for (int i = 1; i < 100; i++) { Block block = _syncingFromBlockTree.FindBlock(i, BlockTreeLookupOptions.None)!; - syncingTooBlockTree.Insert(block.Header); + _syncingToBlockTree.Insert(block.Header); } - Block pivot = _syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!; + _pivotBlock = _syncingFromBlockTree.FindBlock(99, BlockTreeLookupOptions.None)!; - SyncConfig syncConfig = new SyncConfig() + _syncConfig = new SyncConfig() { FastSync = true, - PivotHash = pivot.Hash!.ToString(), - PivotNumber = pivot.Number.ToString(), + PivotHash = _pivotBlock.Hash!.ToString(), + PivotNumber = _pivotBlock.Number.ToString(), AncientBodiesBarrier = 0, FastBlocks = true, DownloadBodiesInFastSync = true, }; - _syncFeed = new BodiesSyncFeed( - syncingTooBlockTree, + _feed = new BodiesSyncFeed( + MainnetSpecProvider.Instance, + _syncingToBlockTree, Substitute.For(), - syncConfig, + _syncConfig, new NullSyncReport(), _blocksDb, + _metadataDb, LimboLogs.Instance, flushDbInterval: 10 ); - _syncFeed.InitializeFeed(); } [Test] public async Task ShouldCallFlushPeriodically() { - BodiesSyncBatch req = (await _syncFeed.PrepareRequest())!; + _feed.InitializeFeed(); + BodiesSyncBatch req = (await _feed.PrepareRequest())!; _blocksDb.FlushCount.Should().Be(1); async Task HandleAndPrepareNextRequest() @@ -79,8 +89,8 @@ async Task HandleAndPrepareNextRequest() req.Response = new OwnedBlockBodies(req.Infos.Take(8).Select((info) => _syncingFromBlockTree.FindBlock(info!.BlockNumber, BlockTreeLookupOptions.None)!.Body).ToArray()); - _syncFeed.HandleResponse(req); - req = (await _syncFeed.PrepareRequest())!; + _feed.HandleResponse(req); + req = (await _feed.PrepareRequest())!; } await HandleAndPrepareNextRequest(); @@ -99,7 +109,8 @@ async Task HandleAndPrepareNextRequest() [Test] public async Task ShouldRecoverOnInsertFailure() { - BodiesSyncBatch req = (await _syncFeed.PrepareRequest())!; + _feed.InitializeFeed(); + BodiesSyncBatch req = (await _feed.PrepareRequest())!; req.Response = new OwnedBlockBodies(req.Infos.Take(8).Select((info) => _syncingFromBlockTree.FindBlock(info!.BlockNumber, BlockTreeLookupOptions.None)!.Body).ToArray()); @@ -115,11 +126,53 @@ public async Task ShouldRecoverOnInsertFailure() return true; }; - Func act = () => _syncFeed.HandleResponse(req); + Func act = () => _feed.HandleResponse(req); act.Should().Throw(); - req = (await _syncFeed.PrepareRequest())!; + req = (await _feed.PrepareRequest())!; req.Infos[0]!.BlockNumber.Should().Be(95); } + + [TestCase(100, false, null, false)] + [TestCase(11052930, false, null, true)] + [TestCase(11052984, false, null, true)] + [TestCase(11052985, false, null, false)] + [TestCase(100, false, 11052984, false)] + [TestCase(11052930, false, 11052984, true)] + [TestCase(11052984, false, 11052984, true)] + [TestCase(11052985, false, 11052984, false)] + [TestCase(100, true, null, false)] + [TestCase(11052930, true, null, false)] + [TestCase(11052984, true, null, false)] + [TestCase(11052985, true, null, false)] + [TestCase(100, false, 0, false)] + [TestCase(11052930, false, 0, false)] + [TestCase(11052984, false, 0, false)] + [TestCase(11052985, false, 0, false)] + public async Task When_finished_sync_with_old_default_barrier_then_finishes_imedietely( + long? lowestInsertedBlockNumber, + bool JustStarted, + long? previousBarrierInDb, + bool shouldfinish) + { + _syncConfig.AncientReceiptsBarrier = 0; + _syncingToBlockTree.LowestInsertedBodyNumber = JustStarted ? _pivotBlock.Number : _pivotBlock.Number - 1; + if (previousBarrierInDb != null) + _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, previousBarrierInDb.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + _feed.InitializeFeed(); + _syncingToBlockTree.LowestInsertedBodyNumber = lowestInsertedBlockNumber; + + BodiesSyncBatch? request = await _feed.PrepareRequest(); + if (shouldfinish) + { + request.Should().BeNull(); + _feed.CurrentState.Should().Be(SyncFeedState.Finished); + } + else + { + request.Should().NotBeNull(); + _feed.CurrentState.Should().NotBe(SyncFeedState.Finished); + } + } } diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs index f3d38201f7e..305727b076b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs @@ -12,8 +12,11 @@ using Nethermind.Consensus; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; +using Nethermind.Core.Test; using Nethermind.Core.Test.Builders; +using Nethermind.Db; using Nethermind.Logging; using Nethermind.Specs; using Nethermind.Specs.Forks; @@ -74,6 +77,7 @@ public Scenario(ISpecProvider specProvider, int nonEmptyBlocks, int txPerBlock, private ISyncConfig _syncConfig = null!; private ISyncReport _syncReport = null!; private IBlockTree _blockTree = null!; + private IDb _metadataDb = null!; private static readonly long _pivotNumber = 1024; @@ -99,6 +103,7 @@ public void Setup() { _receiptStorage = Substitute.For(); _blockTree = Substitute.For(); + _metadataDb = new TestMemDb(); _syncConfig = new SyncConfig { FastBlocks = true, FastSync = true }; _syncConfig.PivotNumber = _pivotNumber.ToString(); @@ -124,6 +129,7 @@ private ReceiptsSyncFeed CreateFeed() _syncPeerPool, _syncConfig, _syncReport, + _metadataDb, LimboLogs.Instance); } @@ -139,6 +145,7 @@ public void Should_throw_when_fast_blocks_not_enabled() _syncPeerPool, _syncConfig, _syncReport, + _metadataDb, LimboLogs.Instance)); } @@ -152,6 +159,7 @@ public async Task Should_finish_on_start_when_receipts_not_stored() _syncPeerPool, _syncConfig, _syncReport, + _metadataDb, LimboLogs.Instance); _feed.InitializeFeed(); @@ -227,14 +235,32 @@ public async Task When_configured_to_skip_receipts_then_finishes_immediately() _measuredProgressQueue.HasEnded.Should().BeTrue(); } - [TestCase(100, false)] - [TestCase(11052930, true)] - [TestCase(11052984, true)] - [TestCase(11052985, false)] + [TestCase(100, false, null, false)] + [TestCase(11052930, false, null, true)] + [TestCase(11052984, false, null, true)] + [TestCase(11052985, false, null, false)] + [TestCase(100, false, 11052984, false)] + [TestCase(11052930, false, 11052984, true)] + [TestCase(11052984, false, 11052984, true)] + [TestCase(11052985, false, 11052984, false)] + [TestCase(100, true, null, false)] + [TestCase(11052930, true, null, false)] + [TestCase(11052984, true, null, false)] + [TestCase(11052985, true, null, false)] + [TestCase(100, false, 0, false)] + [TestCase(11052930, false, 0, false)] + [TestCase(11052984, false, 0, false)] + [TestCase(11052985, false, 0, false)] public async Task When_finished_sync_with_old_default_barrier_then_finishes_imedietely( - long? lowestInsertedReceiptBlockNumber, bool shouldfinish) + long? lowestInsertedReceiptBlockNumber, + bool JustStarted, + long? previousBarrierInDb, + bool shouldfinish) { _syncConfig.AncientReceiptsBarrier = 0; + _receiptStorage.HasBlock(Arg.Is(_pivotNumber), Arg.Any()).Returns(!JustStarted); + if (previousBarrierInDb != null) + _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, previousBarrierInDb.Value.ToBigEndianByteArrayWithoutLeadingZeros()); LoadScenario(_256BodiesWithOneTxEach); _receiptStorage.LowestInsertedReceiptBlockNumber.Returns(lowestInsertedReceiptBlockNumber); @@ -271,6 +297,7 @@ private void LoadScenario(Scenario scenario, ISyncConfig syncConfig) _syncPeerPool, _syncConfig, _syncReport, + _metadataDb, LimboLogs.Instance); _feed.InitializeFeed(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs index 4b876b70933..b3a178bf287 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/ReceiptSyncFeedTests.cs @@ -10,6 +10,7 @@ using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Test.Builders; +using Nethermind.Db; using Nethermind.Logging; using Nethermind.Specs; using Nethermind.Synchronization.FastBlocks; @@ -63,6 +64,7 @@ public async Task ShouldRecoverOnInsertFailure() Substitute.For(), syncConfig, new NullSyncReport(), + new MemDb(), LimboLogs.Instance ); syncFeed.InitializeFeed(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs index 65442fcfd91..e29f8d25919 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -151,34 +152,6 @@ public void Is_fast_block_bodies_finished_returns_false_when_blocks_not_download Assert.False(syncProgressResolver.IsFastBlocksBodiesFinished()); } - [TestCase(100, false)] - [TestCase(11052930, true)] - [TestCase(11052984, true)] - [TestCase(11052985, false)] - public void Is_fast_block_bodies_and_receipts_finished_returns_correct_value_when_within_deposit_contract_barrier(long? lowestInsertedReceiptBlockNumber, bool shouldfinish) - { - IBlockTree blockTree = Substitute.For(); - IReceiptStorage receiptStorage = Substitute.For(); - IDb stateDb = new MemDb(); - SyncConfig syncConfig = new() - { - FastBlocks = true, - FastSync = true, - DownloadBodiesInFastSync = true, - DownloadReceiptsInFastSync = true, - PivotNumber = "1", - }; - ProgressTracker progressTracker = new(blockTree, stateDb, LimboLogs.Instance); - - blockTree.LowestInsertedHeader.Returns(Build.A.BlockHeader.WithNumber(1).WithStateRoot(TestItem.KeccakA).TestObject); - blockTree.LowestInsertedBodyNumber.Returns(lowestInsertedReceiptBlockNumber); - receiptStorage.LowestInsertedReceiptBlockNumber.Returns(lowestInsertedReceiptBlockNumber); - - SyncProgressResolver syncProgressResolver = new(blockTree, receiptStorage, stateDb, NullTrieNodeResolver.Instance, progressTracker, syncConfig, LimboLogs.Instance); - Assert.That(shouldfinish, Is.EqualTo(syncProgressResolver.IsFastBlocksBodiesFinished())); - Assert.That(shouldfinish, Is.EqualTo(syncProgressResolver.IsFastBlocksReceiptsFinished())); - } - [Test] public void Is_fast_block_receipts_finished_returns_true_when_receipts_not_downloaded_and_we_do_not_want_to_download_receipts() { diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 7312b6c712f..4aa694c235e 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -8,6 +8,8 @@ using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; using Nethermind.Db; using Nethermind.Logging; using Nethermind.State.Proofs; @@ -30,12 +32,15 @@ public class BodiesSyncFeed : ActivatedSyncFeed private readonly IBlockTree _blockTree; private readonly ISyncConfig _syncConfig; private readonly ISyncReport _syncReport; + private readonly ISpecProvider _specProvider; private readonly ISyncPeerPool _syncPeerPool; private readonly IDbMeta _blocksDb; + private readonly IDb _metadataDb; public override bool IsFinished => (_blockTree.LowestInsertedBodyNumber ?? long.MaxValue) <= _barrier; private long _pivotNumber; private long _barrier; + private long? _barrierWhenStarted; private SyncStatusList _syncStatusList; @@ -44,15 +49,19 @@ public class BodiesSyncFeed : ActivatedSyncFeed || WithinOldBarrierDefault; // This property was introduced when we switched defaults of barriers from 11052984 to 0 to not disturb existing node operators - private bool WithinOldBarrierDefault => _blockTree.LowestInsertedBodyNumber <= DepositContractBarrier - && _blockTree.LowestInsertedBodyNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; + private bool WithinOldBarrierDefault => _specProvider.ChainId == BlockchainIds.Mainnet + && _barrierWhenStarted == DepositContractBarrier + && _blockTree.LowestInsertedBodyNumber <= DepositContractBarrier + && _blockTree.LowestInsertedBodyNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; public BodiesSyncFeed( + ISpecProvider specProvider, IBlockTree blockTree, ISyncPeerPool syncPeerPool, ISyncConfig syncConfig, ISyncReport syncReport, IDbMeta blocksDb, + IDb metadataDb, ILogManager logManager, long flushDbInterval = DefaultFlushDbInterval) { @@ -62,6 +71,8 @@ public class BodiesSyncFeed : ActivatedSyncFeed _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); _syncReport = syncReport ?? throw new ArgumentNullException(nameof(syncReport)); _blocksDb = blocksDb ?? throw new ArgumentNullException(nameof(blocksDb)); + _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb)); _flushDbInterval = flushDbInterval; if (!_syncConfig.FastBlocks) @@ -81,8 +92,27 @@ public override void InitializeFeed() _barrier = _syncConfig.AncientBodiesBarrierCalc; if (_logger.IsInfo) _logger.Info($"Changed pivot in bodies sync. Now using pivot {_pivotNumber} and barrier {_barrier}"); ResetSyncStatusList(); + if (_blockTree.LowestInsertedBodyNumber is null || _blockTree.LowestInsertedBodyNumber >= _syncConfig.PivotNumberParsed) + { + _barrierWhenStarted = _syncConfig.AncientBodiesBarrierCalc; + _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + else if (_metadataDb.KeyExists(MetadataDbKeys.BodiesBarrierWhenStarted)) + { + _barrierWhenStarted = _metadataDb.Get(MetadataDbKeys.BodiesBarrierWhenStarted).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); + } + else if (_specProvider.ChainId == BlockchainIds.Mainnet) + { + // Assume the bodies barrier was the previous defualt (deposit contract barrier) only for mainnet + _barrierWhenStarted = DepositContractBarrier; + _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + else + { + _barrierWhenStarted = _barrier; + _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } } - base.InitializeFeed(); } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 43162079004..39c34e03f9a 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -50,7 +50,8 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed || WithinOldBarrierDefault; // This property was introduced when we switched defaults of barriers on mainnet from 11052984 to 0 to not disturb existing node operators - private bool WithinOldBarrierDefault => _barrierWhenStarted == DepositContractBarrier + private bool WithinOldBarrierDefault => _specProvider.ChainId == BlockchainIds.Mainnet + && _barrierWhenStarted == DepositContractBarrier && _receiptStorage.LowestInsertedReceiptBlockNumber <= DepositContractBarrier && _receiptStorage.LowestInsertedReceiptBlockNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; // this is intentional. using this as an approxamation assuming a minimum of 1 receipt in per block @@ -90,8 +91,6 @@ public override void InitializeFeed() _barrier = _syncConfig.AncientReceiptsBarrierCalc; if (_logger.IsInfo) _logger.Info($"Changed pivot in receipts sync. Now using pivot {_pivotNumber} and barrier {_barrier}"); ResetSyncStatusList(); - if (_specProvider.ChainId != BlockchainIds.Mainnet) - return; if (!_receiptStorage.HasBlock(_syncConfig.PivotNumberParsed, _syncConfig.PivotHashParsed)) { _barrierWhenStarted = _syncConfig.AncientReceiptsBarrierCalc; @@ -101,10 +100,16 @@ public override void InitializeFeed() { _barrierWhenStarted = _metadataDb.Get(MetadataDbKeys.ReceiptsBarrierWhenStarted).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); } - else + else if (_specProvider.ChainId == BlockchainIds.Mainnet) { - // Assume the receipts barrier was the previous defualt + // Assume the receipts barrier was the previous defualt (deposit contract barrier) only for mainnet _barrierWhenStarted = DepositContractBarrier; + _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + else + { + _barrierWhenStarted = _barrier; + _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); } } diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 49fd21feec7..33361381b43 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -190,13 +190,13 @@ public virtual void Start() private BodiesSyncFeed? CreateBodiesSyncFeed() { if (!_syncConfig.FastSync || !_syncConfig.FastBlocks || !_syncConfig.DownloadHeadersInFastSync || !_syncConfig.DownloadBodiesInFastSync) return null; - return new BodiesSyncFeed(_blockTree, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.BlocksDb, _logManager); + return new BodiesSyncFeed(_specProvider, _blockTree, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.BlocksDb, _dbProvider.MetadataDb, _logManager); } private ReceiptsSyncFeed? CreateReceiptsSyncFeed() { if (!_syncConfig.FastSync || !_syncConfig.FastBlocks || !_syncConfig.DownloadHeadersInFastSync || !_syncConfig.DownloadBodiesInFastSync || !_syncConfig.DownloadReceiptsInFastSync) return null; - return new ReceiptsSyncFeed(_specProvider, _blockTree, _receiptStorage, _syncPeerPool, _syncConfig, _syncReport, _logManager); + return new ReceiptsSyncFeed(_specProvider, _blockTree, _receiptStorage, _syncPeerPool, _syncConfig, _syncReport, _dbProvider.MetadataDb, _logManager); } private SnapSyncFeed? CreateSnapSyncFeed() From ca216417be9d1af02ded6fd8bc753b1154e6e29a Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Fri, 3 Nov 2023 20:03:50 +0300 Subject: [PATCH 11/14] remove useless using --- .../Nethermind.Synchronization.Test/SyncProgressResolverTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs index e29f8d25919..3fa95df3df1 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncProgressResolverTests.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Blockchain; -using Nethermind.Blockchain.Receipts; using Nethermind.Blockchain.Synchronization; using Nethermind.Core; using Nethermind.Core.Crypto; From e25b7d2e5c2972b67c89c3e7ad6da85f385c799e Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Sat, 4 Nov 2023 16:01:05 +0300 Subject: [PATCH 12/14] Fix bodies not finishing --- .../Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs | 4 ++-- .../Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 4aa694c235e..145d569258b 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -36,7 +36,6 @@ public class BodiesSyncFeed : ActivatedSyncFeed private readonly ISyncPeerPool _syncPeerPool; private readonly IDbMeta _blocksDb; private readonly IDb _metadataDb; - public override bool IsFinished => (_blockTree.LowestInsertedBodyNumber ?? long.MaxValue) <= _barrier; private long _pivotNumber; private long _barrier; @@ -45,7 +44,7 @@ public class BodiesSyncFeed : ActivatedSyncFeed private SyncStatusList _syncStatusList; private bool ShouldFinish => !_syncConfig.DownloadBodiesInFastSync || AllDownloaded; - private bool AllDownloaded => _blockTree.LowestInsertedBodyNumber <= _barrier + private bool AllDownloaded => (_blockTree.LowestInsertedBodyNumber ?? long.MaxValue) <= _barrier || WithinOldBarrierDefault; // This property was introduced when we switched defaults of barriers from 11052984 to 0 to not disturb existing node operators @@ -54,6 +53,7 @@ public class BodiesSyncFeed : ActivatedSyncFeed && _blockTree.LowestInsertedBodyNumber <= DepositContractBarrier && _blockTree.LowestInsertedBodyNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; + public override bool IsFinished => AllDownloaded; public BodiesSyncFeed( ISpecProvider specProvider, IBlockTree blockTree, diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 39c34e03f9a..79f8fcfc3a1 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -46,7 +46,7 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed private long? _barrierWhenStarted; private bool ShouldFinish => !_syncConfig.DownloadReceiptsInFastSync || AllDownloaded; - private bool AllDownloaded => _receiptStorage.LowestInsertedReceiptBlockNumber <= _barrier + private bool AllDownloaded => (_receiptStorage.LowestInsertedReceiptBlockNumber ?? long.MaxValue) <= _barrier || WithinOldBarrierDefault; // This property was introduced when we switched defaults of barriers on mainnet from 11052984 to 0 to not disturb existing node operators From 93f380c2765d1841a876d53d2a52f9d60172ce53 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 29 Nov 2023 01:30:28 +0300 Subject: [PATCH 13/14] refactor and minor fixes --- .../FastBlocks/BodiesSyncFeedTests.cs | 12 ++-- .../ReceiptsSyncFeedTests.cs | 16 ++--- .../FastBlocks/BarrierSyncFeed.cs | 67 +++++++++++++++++++ .../FastBlocks/BodiesSyncFeed.cs | 47 +++---------- .../FastBlocks/ReceiptsSyncFeed.cs | 48 +++---------- 5 files changed, 96 insertions(+), 94 deletions(-) rename src/Nethermind/Nethermind.Synchronization.Test/{FastSync => FastBlocks}/ReceiptsSyncFeedTests.cs (97%) create mode 100644 src/Nethermind/Nethermind.Synchronization/FastBlocks/BarrierSyncFeed.cs diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs index 324eb0c8ae6..f4da5c50c37 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs @@ -120,9 +120,7 @@ public async Task ShouldRecoverOnInsertFailure() { writeCount++; if (writeCount == 5) - { throw new Exception("test failure"); - } return true; }; @@ -134,19 +132,19 @@ public async Task ShouldRecoverOnInsertFailure() req.Infos[0]!.BlockNumber.Should().Be(95); } - [TestCase(100, false, null, false)] + [TestCase(99, false, null, false)] [TestCase(11052930, false, null, true)] [TestCase(11052984, false, null, true)] [TestCase(11052985, false, null, false)] - [TestCase(100, false, 11052984, false)] + [TestCase(99, false, 11052984, false)] [TestCase(11052930, false, 11052984, true)] [TestCase(11052984, false, 11052984, true)] [TestCase(11052985, false, 11052984, false)] - [TestCase(100, true, null, false)] + [TestCase(99, true, null, false)] [TestCase(11052930, true, null, false)] [TestCase(11052984, true, null, false)] [TestCase(11052985, true, null, false)] - [TestCase(100, false, 0, false)] + [TestCase(99, false, 0, false)] [TestCase(11052930, false, 0, false)] [TestCase(11052984, false, 0, false)] [TestCase(11052985, false, 0, false)] @@ -157,7 +155,7 @@ public async Task ShouldRecoverOnInsertFailure() bool shouldfinish) { _syncConfig.AncientReceiptsBarrier = 0; - _syncingToBlockTree.LowestInsertedBodyNumber = JustStarted ? _pivotBlock.Number : _pivotBlock.Number - 1; + _syncingToBlockTree.LowestInsertedBodyNumber = JustStarted ? null : _pivotBlock.Number; if (previousBarrierInDb != null) _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, previousBarrierInDb.Value.ToBigEndianByteArrayWithoutLeadingZeros()); _feed.InitializeFeed(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs similarity index 97% rename from src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs rename to src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs index 305727b076b..8ea3a38697b 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastSync/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs @@ -28,7 +28,7 @@ using NSubstitute; using NUnit.Framework; -namespace Nethermind.Synchronization.Test.FastSync +namespace Nethermind.Synchronization.Test.FastBlocks { [TestFixture] public class ReceiptsSyncFeedTests @@ -48,14 +48,10 @@ public Scenario(ISpecProvider specProvider, int nonEmptyBlocks, int txPerBlock, .WithTransactions(blockNumber > _pivotNumber - nonEmptyBlocks ? txPerBlock : 0, specProvider).TestObject; if (blockNumber > _pivotNumber - nonEmptyBlocks - emptyBlocks) - { Blocks[blockNumber] = block; - } if (blockNumber == _pivotNumber - nonEmptyBlocks - emptyBlocks + 1) - { LowestInsertedBody = block; - } parent = block; } @@ -235,19 +231,19 @@ public async Task When_configured_to_skip_receipts_then_finishes_immediately() _measuredProgressQueue.HasEnded.Should().BeTrue(); } - [TestCase(100, false, null, false)] + [TestCase(1024, false, null, false)] [TestCase(11052930, false, null, true)] [TestCase(11052984, false, null, true)] [TestCase(11052985, false, null, false)] - [TestCase(100, false, 11052984, false)] + [TestCase(1024, false, 11052984, false)] [TestCase(11052930, false, 11052984, true)] [TestCase(11052984, false, 11052984, true)] [TestCase(11052985, false, 11052984, false)] - [TestCase(100, true, null, false)] + [TestCase(1024, true, null, false)] [TestCase(11052930, true, null, false)] [TestCase(11052984, true, null, false)] [TestCase(11052985, true, null, false)] - [TestCase(100, false, 0, false)] + [TestCase(1024, false, 0, false)] [TestCase(11052930, false, 0, false)] [TestCase(11052984, false, 0, false)] [TestCase(11052985, false, 0, false)] @@ -307,9 +303,7 @@ private void LoadScenario(Scenario scenario, ISyncConfig syncConfig) { Block? block = scenario.Blocks[ci.Arg()]; if (block is null) - { return null; - } BlockInfo blockInfo = new(block.Hash!, block.TotalDifficulty ?? 0) { diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BarrierSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BarrierSyncFeed.cs new file mode 100644 index 00000000000..a3eee67497a --- /dev/null +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BarrierSyncFeed.cs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Db; +using Nethermind.Logging; +using Nethermind.Synchronization.ParallelSync; +using Nethermind.Synchronization.SyncLimits; + +namespace Nethermind.Synchronization.FastBlocks; + +public abstract class BarrierSyncFeed : ActivatedSyncFeed +{ + internal const int DepositContractBarrier = 11052984; + protected abstract long? LowestInsertedNumber { get; } + protected abstract int BarrierWhenStartedMetadataDbKey { get; } + protected abstract long SyncConfigBarrierCalc { get; } + protected abstract Func HasPivot { get; } + + protected readonly ISpecProvider _specProvider; + protected readonly ILogger _logger; + protected long _barrier; + protected long _pivotNumber; + protected long? _barrierWhenStarted; + + protected readonly IDb _metadataDb; + + // This property was introduced when we switched defaults of barriers on mainnet from 11052984 to 0 to not disturb existing node operators + protected bool WithinOldBarrierDefault => _specProvider.ChainId == BlockchainIds.Mainnet + && _barrierWhenStarted == DepositContractBarrier + && LowestInsertedNumber <= DepositContractBarrier + && LowestInsertedNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; // this is intentional. using this as an approxamation assuming a minimum of 1 receipt in per block in case of receipts + + public BarrierSyncFeed(IDb metadataDb, ISpecProvider specProvider, ILogger logger) + { + _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb)); + _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + public void InitializeMetadataDb() + { + if (!HasPivot()) + { + _barrierWhenStarted = SyncConfigBarrierCalc; + _metadataDb.Set(BarrierWhenStartedMetadataDbKey, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + else if (_metadataDb.KeyExists(BarrierWhenStartedMetadataDbKey)) + { + _barrierWhenStarted = _metadataDb.Get(BarrierWhenStartedMetadataDbKey).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); + } + else if (_specProvider.ChainId == BlockchainIds.Mainnet) + { + // Assume the barrier was the previous defualt (deposit contract barrier) only for mainnet + _barrierWhenStarted = DepositContractBarrier; + _metadataDb.Set(BarrierWhenStartedMetadataDbKey, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + else + { + _barrierWhenStarted = _barrier; + _metadataDb.Set(BarrierWhenStartedMetadataDbKey, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); + } + } +} diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs index 145d569258b..62803ba7f57 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/BodiesSyncFeed.cs @@ -21,25 +21,23 @@ namespace Nethermind.Synchronization.FastBlocks { - public class BodiesSyncFeed : ActivatedSyncFeed + public class BodiesSyncFeed : BarrierSyncFeed { - internal const int DepositContractBarrier = 11052984; + protected override long? LowestInsertedNumber => _blockTree.LowestInsertedBodyNumber; + protected override int BarrierWhenStartedMetadataDbKey => MetadataDbKeys.BodiesBarrierWhenStarted; + protected override long SyncConfigBarrierCalc => _syncConfig.AncientBodiesBarrierCalc; + protected override Func HasPivot => + () => _blockTree.LowestInsertedBodyNumber is not null && _blockTree.LowestInsertedBodyNumber <= _syncConfig.PivotNumberParsed; + private int _requestSize = GethSyncLimits.MaxBodyFetch; private const long DefaultFlushDbInterval = 100000; // About every 10GB on mainnet private readonly long _flushDbInterval; // About every 10GB on mainnet - private readonly ILogger _logger; private readonly IBlockTree _blockTree; private readonly ISyncConfig _syncConfig; private readonly ISyncReport _syncReport; - private readonly ISpecProvider _specProvider; private readonly ISyncPeerPool _syncPeerPool; private readonly IDbMeta _blocksDb; - private readonly IDb _metadataDb; - - private long _pivotNumber; - private long _barrier; - private long? _barrierWhenStarted; private SyncStatusList _syncStatusList; @@ -47,12 +45,6 @@ public class BodiesSyncFeed : ActivatedSyncFeed private bool AllDownloaded => (_blockTree.LowestInsertedBodyNumber ?? long.MaxValue) <= _barrier || WithinOldBarrierDefault; - // This property was introduced when we switched defaults of barriers from 11052984 to 0 to not disturb existing node operators - private bool WithinOldBarrierDefault => _specProvider.ChainId == BlockchainIds.Mainnet - && _barrierWhenStarted == DepositContractBarrier - && _blockTree.LowestInsertedBodyNumber <= DepositContractBarrier - && _blockTree.LowestInsertedBodyNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; - public override bool IsFinished => AllDownloaded; public BodiesSyncFeed( ISpecProvider specProvider, @@ -64,15 +56,13 @@ public class BodiesSyncFeed : ActivatedSyncFeed IDb metadataDb, ILogManager logManager, long flushDbInterval = DefaultFlushDbInterval) + : base(metadataDb, specProvider, logManager.GetClassLogger()) { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); _syncPeerPool = syncPeerPool ?? throw new ArgumentNullException(nameof(syncPeerPool)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); _syncReport = syncReport ?? throw new ArgumentNullException(nameof(syncReport)); _blocksDb = blocksDb ?? throw new ArgumentNullException(nameof(blocksDb)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); - _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb)); _flushDbInterval = flushDbInterval; if (!_syncConfig.FastBlocks) @@ -92,26 +82,7 @@ public override void InitializeFeed() _barrier = _syncConfig.AncientBodiesBarrierCalc; if (_logger.IsInfo) _logger.Info($"Changed pivot in bodies sync. Now using pivot {_pivotNumber} and barrier {_barrier}"); ResetSyncStatusList(); - if (_blockTree.LowestInsertedBodyNumber is null || _blockTree.LowestInsertedBodyNumber >= _syncConfig.PivotNumberParsed) - { - _barrierWhenStarted = _syncConfig.AncientBodiesBarrierCalc; - _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } - else if (_metadataDb.KeyExists(MetadataDbKeys.BodiesBarrierWhenStarted)) - { - _barrierWhenStarted = _metadataDb.Get(MetadataDbKeys.BodiesBarrierWhenStarted).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); - } - else if (_specProvider.ChainId == BlockchainIds.Mainnet) - { - // Assume the bodies barrier was the previous defualt (deposit contract barrier) only for mainnet - _barrierWhenStarted = DepositContractBarrier; - _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } - else - { - _barrierWhenStarted = _barrier; - _metadataDb.Set(MetadataDbKeys.BodiesBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } + InitializeMetadataDb(); } base.InitializeFeed(); } diff --git a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs index 79f8fcfc3a1..bd5620f616d 100644 --- a/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs +++ b/src/Nethermind/Nethermind.Synchronization/FastBlocks/ReceiptsSyncFeed.cs @@ -26,36 +26,30 @@ namespace Nethermind.Synchronization.FastBlocks { - public class ReceiptsSyncFeed : ActivatedSyncFeed + public class ReceiptsSyncFeed : BarrierSyncFeed { - internal const int DepositContractBarrier = 11052984; + protected override long? LowestInsertedNumber => _receiptStorage.LowestInsertedReceiptBlockNumber; + protected override int BarrierWhenStartedMetadataDbKey => MetadataDbKeys.ReceiptsBarrierWhenStarted; + protected override long SyncConfigBarrierCalc => _syncConfig.AncientReceiptsBarrierCalc; + protected override Func HasPivot => + () => _receiptStorage.HasBlock(_syncConfig.PivotNumberParsed, _syncConfig.PivotHashParsed); + private int _requestSize = GethSyncLimits.MaxReceiptFetch; - private readonly ILogger _logger; private readonly IBlockTree _blockTree; private readonly ISyncConfig _syncConfig; private readonly ISyncReport _syncReport; - private readonly ISpecProvider _specProvider; private readonly IReceiptStorage _receiptStorage; private readonly ISyncPeerPool _syncPeerPool; - private readonly IDb _metadataDb; private SyncStatusList _syncStatusList; - private long _pivotNumber; - private long _barrier; - private long? _barrierWhenStarted; private bool ShouldFinish => !_syncConfig.DownloadReceiptsInFastSync || AllDownloaded; private bool AllDownloaded => (_receiptStorage.LowestInsertedReceiptBlockNumber ?? long.MaxValue) <= _barrier || WithinOldBarrierDefault; - // This property was introduced when we switched defaults of barriers on mainnet from 11052984 to 0 to not disturb existing node operators - private bool WithinOldBarrierDefault => _specProvider.ChainId == BlockchainIds.Mainnet - && _barrierWhenStarted == DepositContractBarrier - && _receiptStorage.LowestInsertedReceiptBlockNumber <= DepositContractBarrier - && _receiptStorage.LowestInsertedReceiptBlockNumber > DepositContractBarrier - GethSyncLimits.MaxBodyFetch; // this is intentional. using this as an approxamation assuming a minimum of 1 receipt in per block - public override bool IsFinished => AllDownloaded; + public ReceiptsSyncFeed( ISpecProvider specProvider, IBlockTree blockTree, @@ -65,15 +59,13 @@ public class ReceiptsSyncFeed : ActivatedSyncFeed ISyncReport syncReport, IDb metadataDb, ILogManager logManager) + : base(metadataDb, specProvider, logManager?.GetClassLogger()) { - _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); _receiptStorage = receiptStorage ?? throw new ArgumentNullException(nameof(receiptStorage)); - _specProvider = specProvider ?? throw new ArgumentNullException(nameof(specProvider)); _syncPeerPool = syncPeerPool ?? throw new ArgumentNullException(nameof(syncPeerPool)); _syncConfig = syncConfig ?? throw new ArgumentNullException(nameof(syncConfig)); _syncReport = syncReport ?? throw new ArgumentNullException(nameof(syncReport)); _blockTree = blockTree ?? throw new ArgumentNullException(nameof(blockTree)); - _metadataDb = metadataDb ?? throw new ArgumentNullException(nameof(metadataDb)); if (!_syncConfig.FastBlocks) { @@ -91,28 +83,8 @@ public override void InitializeFeed() _barrier = _syncConfig.AncientReceiptsBarrierCalc; if (_logger.IsInfo) _logger.Info($"Changed pivot in receipts sync. Now using pivot {_pivotNumber} and barrier {_barrier}"); ResetSyncStatusList(); - if (!_receiptStorage.HasBlock(_syncConfig.PivotNumberParsed, _syncConfig.PivotHashParsed)) - { - _barrierWhenStarted = _syncConfig.AncientReceiptsBarrierCalc; - _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } - else if (_metadataDb.KeyExists(MetadataDbKeys.ReceiptsBarrierWhenStarted)) - { - _barrierWhenStarted = _metadataDb.Get(MetadataDbKeys.ReceiptsBarrierWhenStarted).ToLongFromBigEndianByteArrayWithoutLeadingZeros(); - } - else if (_specProvider.ChainId == BlockchainIds.Mainnet) - { - // Assume the receipts barrier was the previous defualt (deposit contract barrier) only for mainnet - _barrierWhenStarted = DepositContractBarrier; - _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } - else - { - _barrierWhenStarted = _barrier; - _metadataDb.Set(MetadataDbKeys.ReceiptsBarrierWhenStarted, _barrierWhenStarted.Value.ToBigEndianByteArrayWithoutLeadingZeros()); - } + InitializeMetadataDb(); } - base.InitializeFeed(); } From 91d261b7fd64019957897ad83c29634f77724618 Mon Sep 17 00:00:00 2001 From: smartprogrammer Date: Wed, 6 Dec 2023 20:38:33 +0300 Subject: [PATCH 14/14] fix test issue --- .../FastBlocks/BodiesSyncFeedTests.cs | 3 ++- .../FastBlocks/ReceiptsSyncFeedTests.cs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs index 0cc7b60433a..f007a0439d4 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/BodiesSyncFeedTests.cs @@ -81,7 +81,8 @@ public void Setup() public void TearDown() { _blocksDb?.Dispose(); - _syncFeed?.Dispose(); + _feed?.Dispose(); + _metadataDb?.Dispose(); } [Test] diff --git a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs index 60854a4eee0..999be551fa5 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/FastBlocks/ReceiptsSyncFeedTests.cs @@ -122,6 +122,7 @@ public void TearDown() _feed?.Dispose(); _syncPeerPool?.Dispose(); _syncReport?.Dispose(); + _metadataDb?.Dispose(); } private ReceiptsSyncFeed CreateFeed()