From 6f5fe69a99368df67e0a7ef4d2f363bcb6e8710e Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 22 Jan 2021 13:53:35 +0800 Subject: [PATCH 01/91] add header back --- src/neo/Ledger/Blockchain.cs | 72 +++++++++++++++++-- src/neo/Network/P2P/MessageCommand.cs | 8 +-- src/neo/Network/P2P/Payloads/Header.cs | 16 +++++ .../Network/P2P/RemoteNode.ProtocolHandler.cs | 27 +++---- src/neo/Network/P2P/RemoteNode.cs | 4 -- src/neo/Network/P2P/TaskManager.cs | 56 ++++++--------- .../SmartContract/Native/LedgerContract.cs | 25 ++++++- tests/neo.UnitTests/Network/P2P/UT_Message.cs | 4 +- .../Network/P2P/UT_RemoteNodeMailbox.cs | 10 --- 9 files changed, 141 insertions(+), 81 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index e7bb4bcfba..73ca642e5c 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -72,6 +72,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); + public uint HeaderHeight => NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); private static Blockchain singleton; @@ -216,30 +217,83 @@ private VerifyResult OnNewBlock(Block block) { if (block.Index <= Height) return VerifyResult.AlreadyExists; - if (block.Index - 1 > Height) + if (block.Index - 1 > HeaderHeight) { AddUnverifiedBlockToCache(block); return VerifyResult.UnableToVerify; } - if (block.Index == Height + 1) + if (block.Index == HeaderHeight + 1) { if (!block.Verify(currentSnapshot)) return VerifyResult.Invalid; - block_cache.TryAdd(block.Hash, block); - block_cache_unverified.Remove(block.Index); - Persist(block); + } + else + { + if (!block.Hash.Equals(NativeContract.Ledger.GetBlockHash(currentSnapshot, block.Index))) + return VerifyResult.Invalid; + } + block_cache.TryAdd(block.Hash, block); + if (block.Index == Height + 1) + { + Block block_persist = block; + List blocksToPersistList = new List(); + while (true) + { + blocksToPersistList.Add(block_persist); + if (block_persist.Index + 1 >= NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot)) break; + UInt256 hash = NativeContract.Ledger.CurrentHeaderHash(currentSnapshot); + if (!block_cache.TryGetValue(hash, out block_persist)) break; + } + + int blocksPersisted = 0; + foreach (Block blockToPersist in blocksToPersistList) + { + block_cache_unverified.Remove(blockToPersist.Index); + Persist(blockToPersist); + + // 15000 is the default among of seconds per block, while MilliSecondsPerBlock is the current + uint extraBlocks = (15000 - MillisecondsPerBlock) / 1000; + + if (blocksPersisted++ < blocksToPersistList.Count - (2 + Math.Max(0, extraBlocks))) continue; + // Empirically calibrated for relaying the most recent 2 blocks persisted with 15s network + // Increase in the rate of 1 block per second in configurations with faster blocks + + if (blockToPersist.Index + 100 >= NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot)) + system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = blockToPersist }); + } if (block_cache_unverified.TryGetValue(Height + 1, out var unverifiedBlocks)) { foreach (var unverifiedBlock in unverifiedBlocks.Blocks) Self.Tell(unverifiedBlock, ActorRefs.NoSender); block_cache_unverified.Remove(Height + 1); } - // We can store the new block in block_cache and tell the new height to other nodes after Persist(). - system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Singleton.Height))); + } + else + { + if (block.Index + 100 >= NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot)) + system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = block }); } return VerifyResult.Succeed; } + private void OnNewHeaders(Header[] headers) + { + using (SnapshotCache snapshot = GetSnapshot()) + { + foreach (Header header in headers) + { + if (header.Index - 1 >= HeaderHeight) break; + if (header.Index < HeaderHeight) continue; + if (!header.Verify(snapshot)) break; + NativeContract.Ledger.SaveHeader(snapshot, header); + NativeContract.Ledger.SetCurrentHeader(snapshot, header.Hash, header.Index); + } + snapshot.Commit(); + } + UpdateCurrentSnapshot(); + system.TaskManager.Tell(new TaskManager.HeaderTaskCompleted(), Sender); + } + private VerifyResult OnNewInventory(IInventory inventory) { if (!inventory.Verify(currentSnapshot)) return VerifyResult.Invalid; @@ -271,6 +325,9 @@ protected override void OnReceive(object message) case FillMemoryPool fill: OnFillMemoryPool(fill.Transactions); break; + case Header[] headers: + OnNewHeaders(headers); + break; case Block block: OnInventory(block, false); break; @@ -430,6 +487,7 @@ internal protected override bool IsHighPriority(object message) { switch (message) { + case Header[] _: case Block _: case ExtensiblePayload _: case Terminated _: diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index 04856c4a48..036484ff6d 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -14,10 +14,10 @@ public enum MessageCommand : byte GetAddr = 0x10, [ReflectionCache(typeof(AddrPayload))] Addr = 0x11, - [ReflectionCache(typeof(PingPayload))] - Ping = 0x18, - [ReflectionCache(typeof(PingPayload))] - Pong = 0x19, + //[ReflectionCache(typeof(PingPayload))] + //Ping = 0x18, + //[ReflectionCache(typeof(PingPayload))] + //Pong = 0x19, //synchronization [ReflectionCache(typeof(GetBlockByIndexPayload))] diff --git a/src/neo/Network/P2P/Payloads/Header.cs b/src/neo/Network/P2P/Payloads/Header.cs index 301d1cf05b..ffabe7a172 100644 --- a/src/neo/Network/P2P/Payloads/Header.cs +++ b/src/neo/Network/P2P/Payloads/Header.cs @@ -1,3 +1,4 @@ +using Neo.SmartContract.Native; using System; using System.IO; @@ -35,5 +36,20 @@ public override void Serialize(BinaryWriter writer) base.Serialize(writer); writer.Write((byte)0); } + + public TrimmedBlock Trim() + { + return new TrimmedBlock + { + Version = Version, + PrevHash = PrevHash, + MerkleRoot = MerkleRoot, + Timestamp = Timestamp, + Index = Index, + NextConsensus = NextConsensus, + Witness = Witness, + Hashes = new UInt256[0] + }; + } } } diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 0cb6b1afcb..695192d45f 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -90,18 +90,15 @@ private void OnMessage(Message msg) case MessageCommand.GetHeaders: OnGetHeadersMessageReceived((GetBlockByIndexPayload)msg.Payload); break; + case MessageCommand.Headers: + OnHeadersMessageReceived((HeadersPayload)msg.Payload); + break; case MessageCommand.Inv: OnInvMessageReceived((InvPayload)msg.Payload); break; case MessageCommand.Mempool: OnMemPoolMessageReceived(); break; - case MessageCommand.Ping: - OnPingMessageReceived((PingPayload)msg.Payload); - break; - case MessageCommand.Pong: - OnPongMessageReceived((PingPayload)msg.Payload); - break; case MessageCommand.Transaction: if (msg.Payload.Size <= Transaction.MaxTransactionSize) OnInventoryReceived((Transaction)msg.Payload); @@ -110,7 +107,6 @@ private void OnMessage(Message msg) case MessageCommand.Version: throw new ProtocolViolationException(); case MessageCommand.Alert: - case MessageCommand.Headers: case MessageCommand.MerkleBlock: case MessageCommand.NotFound: case MessageCommand.Reject: @@ -290,6 +286,12 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) EnqueueMessage(Message.Create(MessageCommand.Headers, HeadersPayload.Create(headers.ToArray()))); } + private void OnHeadersMessageReceived(HeadersPayload payload) + { + if (payload.Headers.Length == 0) return; + system.Blockchain.Tell(payload.Headers); + } + private void OnInventoryReceived(IInventory inventory) { pendingKnownHashes.Remove(inventory.Hash); @@ -330,17 +332,6 @@ private void OnMemPoolMessageReceived() EnqueueMessage(Message.Create(MessageCommand.Inv, payload)); } - private void OnPingMessageReceived(PingPayload payload) - { - UpdateLastBlockIndex(payload.LastBlockIndex, true); - EnqueueMessage(Message.Create(MessageCommand.Pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); - } - - private void OnPongMessageReceived(PingPayload payload) - { - UpdateLastBlockIndex(payload.LastBlockIndex, true); - } - private void OnVerackMessageReceived() { verack = true; diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index 5fdc088b92..7125c3b003 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -77,8 +77,6 @@ private void EnqueueMessage(Message message) case MessageCommand.GetBlocks: case MessageCommand.GetHeaders: case MessageCommand.Mempool: - case MessageCommand.Ping: - case MessageCommand.Pong: is_single = true; break; } @@ -130,8 +128,6 @@ protected override void OnReceive(object message) { if (payload.LastBlockIndex > LastHeightSent) LastHeightSent = payload.LastBlockIndex; - else if (msg.Command == MessageCommand.Ping) - break; } EnqueueMessage(msg); break; diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 1e11baf4e0..1fc7bb4265 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -19,11 +19,13 @@ public class TaskManager : UntypedActor internal class Register { public VersionPayload Version; } internal class Update { public uint LastBlockIndex; public bool RequestTasks; } internal class NewTasks { public InvPayload Payload; } + public class HeaderTaskCompleted { } public class RestartTasks { public InvPayload Payload; } private class Timer { } private static readonly TimeSpan TimerInterval = TimeSpan.FromSeconds(30); private static readonly TimeSpan TaskTimeout = TimeSpan.FromMinutes(1); + private static readonly UInt256 HeaderTaskHash = UInt256.Zero; private static readonly UInt256 MemPoolTaskHash = UInt256.Parse("0x0000000000000000000000000000000000000000000000000000000000000001"); private const int MaxConncurrentTasks = 3; @@ -40,6 +42,7 @@ private class Timer { } private readonly HashSet failedSyncTasks = new HashSet(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); + private bool HasHeaderTask => globalTasks.ContainsKey(HeaderTaskHash); private uint lastTaskIndex = 0; public TaskManager(NeoSystem system) @@ -51,6 +54,15 @@ public TaskManager(NeoSystem system) Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } + private void OnHeaderTaskCompleted() + { + if (!sessions.TryGetValue(Sender, out TaskSession session)) + return; + session.InvTasks.Remove(HeaderTaskHash); + DecrementGlobalTask(HeaderTaskHash); + RequestTasks(); + } + private bool AssignSyncTask(uint index, TaskSession filterSession = null) { if (index <= Blockchain.Singleton.Height || sessions.Values.Any(p => p != filterSession && p.IndexTasks.ContainsKey(index))) @@ -78,7 +90,7 @@ private void OnBlock(Block block) if (session is null) return; session.IndexTasks.Remove(block.Index); receivedBlockIndex.TryAdd(block.Index, session); - RequestTasks(false); + RequestTasks(); } private void OnInvalidBlock(Block invalidBlock) @@ -121,7 +133,7 @@ private void OnNewTasks(InvPayload payload) private void OnPersistCompleted(Block block) { receivedBlockIndex.Remove(block.Index); - RequestTasks(false); + RequestTasks(); } protected override void OnReceive(object message) @@ -137,6 +149,9 @@ protected override void OnReceive(object message) case NewTasks tasks: OnNewTasks(tasks.Payload); break; + case HeaderTaskCompleted _: + OnHeaderTaskCompleted(); + break; case RestartTasks restart: OnRestartTasks(restart.Payload); break; @@ -169,7 +184,7 @@ private void OnRegister(VersionPayload version) if (session.IsFullNode) session.InvTasks.TryAdd(MemPoolTaskHash, TimeProvider.Current.UtcNow); sessions.TryAdd(Sender, session); - RequestTasks(true); + RequestTasks(); } private void OnUpdate(Update update) @@ -178,7 +193,7 @@ private void OnUpdate(Update update) return; session.LastBlockIndex = update.LastBlockIndex; session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); - if (update.RequestTasks) RequestTasks(true); + if (update.RequestTasks) RequestTasks(); } private void OnRestartTasks(InvPayload payload) @@ -260,7 +275,7 @@ private void OnTimer() } } } - RequestTasks(true); + RequestTasks(); } protected override void PostStop() @@ -274,12 +289,10 @@ public static Props Props(NeoSystem system) return Akka.Actor.Props.Create(() => new TaskManager(system)).WithMailbox("task-manager-mailbox"); } - private void RequestTasks(bool sendPing) + private void RequestTasks() { if (sessions.Count() == 0) return; - if (sendPing) SendPingMessage(); - while (failedSyncTasks.Count() > 0) { var failedTask = failedSyncTasks.First(); @@ -299,33 +312,6 @@ private void RequestTasks(bool sendPing) if (!AssignSyncTask(++lastTaskIndex)) break; } } - - private void SendPingMessage() - { - TrimmedBlock block; - using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) - { - block = NativeContract.Ledger.GetTrimmedBlock(snapshot, NativeContract.Ledger.CurrentHash(snapshot)); - } - - foreach (KeyValuePair item in sessions) - { - var node = item.Key; - var session = item.Value; - - if (session.ExpireTime < TimeProvider.Current.UtcNow || - (block.Index >= session.LastBlockIndex && - TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= block.Timestamp)) - { - if (session.InvTasks.Remove(MemPoolTaskHash)) - { - node.Tell(Message.Create(MessageCommand.Mempool)); - } - node.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Blockchain.Singleton.Height))); - session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); - } - } - } } internal class TaskManagerMailbox : PriorityMailbox diff --git a/src/neo/SmartContract/Native/LedgerContract.cs b/src/neo/SmartContract/Native/LedgerContract.cs index 2658584c7f..834f6f8a04 100644 --- a/src/neo/SmartContract/Native/LedgerContract.cs +++ b/src/neo/SmartContract/Native/LedgerContract.cs @@ -15,6 +15,7 @@ public sealed class LedgerContract : NativeContract private const byte Prefix_CurrentBlock = 12; private const byte Prefix_Block = 5; private const byte Prefix_Transaction = 11; + private const byte Prefix_CurrentHeader = 14; internal LedgerContract() { @@ -23,7 +24,7 @@ internal LedgerContract() internal override void OnPersist(ApplicationEngine engine) { engine.Snapshot.Add(CreateStorageKey(Prefix_BlockHash).AddBigEndian(engine.PersistingBlock.Index), new StorageItem(engine.PersistingBlock.Hash.ToArray(), true)); - engine.Snapshot.Add(CreateStorageKey(Prefix_Block).Add(engine.PersistingBlock.Hash), new StorageItem(Trim(engine.PersistingBlock).ToArray(), true)); + engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_Block).Add(engine.PersistingBlock.Hash), () => new StorageItem(Trim(engine.PersistingBlock).ToArray(), true)); foreach (Transaction tx in engine.PersistingBlock.Transactions) { engine.Snapshot.Add(CreateStorageKey(Prefix_Transaction).Add(tx.Hash), new StorageItem(new TransactionState @@ -152,6 +153,28 @@ public Transaction GetTransaction(DataCache snapshot, UInt256 hash) return GetTransactionState(snapshot, hash)?.Transaction; } + public void SetCurrentHeader(DataCache snapshot, UInt256 hash, uint index) + { + HashIndexState state = snapshot.GetAndChange(CreateStorageKey(Prefix_CurrentHeader), () => new StorageItem(new HashIndexState())).GetInteroperable(); + state.Hash = hash; + state.Index = index; + } + + public UInt256 CurrentHeaderHash(DataCache snapshot) + { + return snapshot[CreateStorageKey(Prefix_CurrentHeader)].GetInteroperable().Hash; + } + + public uint CurrentHeaderIndex(DataCache snapshot) + { + return snapshot[CreateStorageKey(Prefix_CurrentHeader)].GetInteroperable().Index; + } + + public void SaveHeader(DataCache snapshot, Header header) + { + snapshot.Add(CreateStorageKey(Prefix_Block).Add(header.Hash), new StorageItem(header.Trim().ToArray(), true)); + } + [ContractMethod(0_01000000, CallFlags.ReadStates, Name = "getTransaction")] private Transaction GetTransactionForContract(DataCache snapshot, UInt256 hash) { diff --git a/tests/neo.UnitTests/Network/P2P/UT_Message.cs b/tests/neo.UnitTests/Network/P2P/UT_Message.cs index c70e0b37fd..3cb1b33a8e 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_Message.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_Message.cs @@ -16,7 +16,7 @@ public class UT_Message public void Serialize_Deserialize() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Ping, payload); + var msg = Message.Create(MessageCommand.Inv, payload); var buffer = msg.ToArray(); var copy = buffer.AsSerializable(); var payloadCopy = (PingPayload)copy.Payload; @@ -34,7 +34,7 @@ public void Serialize_Deserialize() public void Serialize_Deserialize_ByteString() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Ping, payload); + var msg = Message.Create(MessageCommand.Inv, payload); var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); diff --git a/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs b/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs index a5174d6cfe..297b87f7f6 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs @@ -44,8 +44,6 @@ public void RemoteNode_Test_IsHighPriority() //connectivity uut.IsHighPriority(Message.Create(MessageCommand.GetAddr, s)).Should().Be(false); uut.IsHighPriority(Message.Create(MessageCommand.Addr, s)).Should().Be(false); - uut.IsHighPriority(Message.Create(MessageCommand.Ping, s)).Should().Be(false); - uut.IsHighPriority(Message.Create(MessageCommand.Pong, s)).Should().Be(false); //synchronization uut.IsHighPriority(Message.Create(MessageCommand.GetHeaders, s)).Should().Be(false); @@ -110,14 +108,6 @@ public void ProtocolHandlerMailbox_Test_ShallDrop() msg = Message.Create(MessageCommand.Addr, s); uut.ShallDrop(msg, emptyQueue).Should().Be(false); uut.ShallDrop(msg, new object[] { msg }).Should().Be(false); - // Ping (no drop) - msg = Message.Create(MessageCommand.Ping, s); - uut.ShallDrop(msg, emptyQueue).Should().Be(false); - uut.ShallDrop(msg, new object[] { msg }).Should().Be(false); - // Pong (no drop) - msg = Message.Create(MessageCommand.Pong, s); - uut.ShallDrop(msg, emptyQueue).Should().Be(false); - uut.ShallDrop(msg, new object[] { msg }).Should().Be(false); //synchronization // GetHeaders (drop) From ce6e47939757366fb299fae3535c1e2729c4eecd Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 22 Jan 2021 13:56:50 +0800 Subject: [PATCH 02/91] remove ping pong --- src/neo/Network/P2P/MessageCommand.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index 036484ff6d..4c0decf512 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -14,10 +14,6 @@ public enum MessageCommand : byte GetAddr = 0x10, [ReflectionCache(typeof(AddrPayload))] Addr = 0x11, - //[ReflectionCache(typeof(PingPayload))] - //Ping = 0x18, - //[ReflectionCache(typeof(PingPayload))] - //Pong = 0x19, //synchronization [ReflectionCache(typeof(GetBlockByIndexPayload))] From af9d2a8d626e1bdd17396b55fc39c4ccf0511410 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 22 Jan 2021 16:53:42 +0800 Subject: [PATCH 03/91] restore ping pong --- src/neo/Ledger/Blockchain.cs | 2 + src/neo/Network/P2P/MessageCommand.cs | 4 ++ .../Network/P2P/RemoteNode.ProtocolHandler.cs | 17 ++++++++ src/neo/Network/P2P/RemoteNode.cs | 4 ++ src/neo/Network/P2P/TaskManager.cs | 43 ++++++++++++++++--- .../Network/P2P/UT_RemoteNodeMailbox.cs | 10 +++++ 6 files changed, 73 insertions(+), 7 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 73ca642e5c..4e5fbab19b 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -267,6 +267,8 @@ private VerifyResult OnNewBlock(Block block) Self.Tell(unverifiedBlock, ActorRefs.NoSender); block_cache_unverified.Remove(Height + 1); } + // We can store the new block in block_cache and tell the new height to other nodes after Persist(). + system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Singleton.Height))); } else { diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index 4c0decf512..04856c4a48 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -14,6 +14,10 @@ public enum MessageCommand : byte GetAddr = 0x10, [ReflectionCache(typeof(AddrPayload))] Addr = 0x11, + [ReflectionCache(typeof(PingPayload))] + Ping = 0x18, + [ReflectionCache(typeof(PingPayload))] + Pong = 0x19, //synchronization [ReflectionCache(typeof(GetBlockByIndexPayload))] diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 695192d45f..5bf07520c9 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -99,6 +99,12 @@ private void OnMessage(Message msg) case MessageCommand.Mempool: OnMemPoolMessageReceived(); break; + case MessageCommand.Ping: + OnPingMessageReceived((PingPayload)msg.Payload); + break; + case MessageCommand.Pong: + OnPongMessageReceived((PingPayload)msg.Payload); + break; case MessageCommand.Transaction: if (msg.Payload.Size <= Transaction.MaxTransactionSize) OnInventoryReceived((Transaction)msg.Payload); @@ -332,6 +338,17 @@ private void OnMemPoolMessageReceived() EnqueueMessage(Message.Create(MessageCommand.Inv, payload)); } + private void OnPingMessageReceived(PingPayload payload) + { + UpdateLastBlockIndex(payload.LastBlockIndex, true); + EnqueueMessage(Message.Create(MessageCommand.Pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); + } + + private void OnPongMessageReceived(PingPayload payload) + { + UpdateLastBlockIndex(payload.LastBlockIndex, true); + } + private void OnVerackMessageReceived() { verack = true; diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index 7125c3b003..5fdc088b92 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -77,6 +77,8 @@ private void EnqueueMessage(Message message) case MessageCommand.GetBlocks: case MessageCommand.GetHeaders: case MessageCommand.Mempool: + case MessageCommand.Ping: + case MessageCommand.Pong: is_single = true; break; } @@ -128,6 +130,8 @@ protected override void OnReceive(object message) { if (payload.LastBlockIndex > LastHeightSent) LastHeightSent = payload.LastBlockIndex; + else if (msg.Command == MessageCommand.Ping) + break; } EnqueueMessage(msg); break; diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 1fc7bb4265..1bb8d4176a 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -60,7 +60,7 @@ private void OnHeaderTaskCompleted() return; session.InvTasks.Remove(HeaderTaskHash); DecrementGlobalTask(HeaderTaskHash); - RequestTasks(); + RequestTasks(true); } private bool AssignSyncTask(uint index, TaskSession filterSession = null) @@ -90,7 +90,7 @@ private void OnBlock(Block block) if (session is null) return; session.IndexTasks.Remove(block.Index); receivedBlockIndex.TryAdd(block.Index, session); - RequestTasks(); + RequestTasks(false); } private void OnInvalidBlock(Block invalidBlock) @@ -133,7 +133,7 @@ private void OnNewTasks(InvPayload payload) private void OnPersistCompleted(Block block) { receivedBlockIndex.Remove(block.Index); - RequestTasks(); + RequestTasks(false); } protected override void OnReceive(object message) @@ -184,7 +184,7 @@ private void OnRegister(VersionPayload version) if (session.IsFullNode) session.InvTasks.TryAdd(MemPoolTaskHash, TimeProvider.Current.UtcNow); sessions.TryAdd(Sender, session); - RequestTasks(); + RequestTasks(true); } private void OnUpdate(Update update) @@ -193,7 +193,7 @@ private void OnUpdate(Update update) return; session.LastBlockIndex = update.LastBlockIndex; session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); - if (update.RequestTasks) RequestTasks(); + if (update.RequestTasks) RequestTasks(true); } private void OnRestartTasks(InvPayload payload) @@ -275,7 +275,7 @@ private void OnTimer() } } } - RequestTasks(); + RequestTasks(true); } protected override void PostStop() @@ -289,10 +289,12 @@ public static Props Props(NeoSystem system) return Akka.Actor.Props.Create(() => new TaskManager(system)).WithMailbox("task-manager-mailbox"); } - private void RequestTasks() + private void RequestTasks(bool sendPing) { if (sessions.Count() == 0) return; + if (sendPing) SendPingMessage(); + while (failedSyncTasks.Count() > 0) { var failedTask = failedSyncTasks.First(); @@ -312,6 +314,33 @@ private void RequestTasks() if (!AssignSyncTask(++lastTaskIndex)) break; } } + + private void SendPingMessage() + { + TrimmedBlock block; + using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) + { + block = NativeContract.Ledger.GetTrimmedBlock(snapshot, NativeContract.Ledger.CurrentHash(snapshot)); + } + + foreach (KeyValuePair item in sessions) + { + var node = item.Key; + var session = item.Value; + + if (session.ExpireTime < TimeProvider.Current.UtcNow || + (block.Index >= session.LastBlockIndex && + TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= block.Timestamp)) + { + if (session.InvTasks.Remove(MemPoolTaskHash)) + { + node.Tell(Message.Create(MessageCommand.Mempool)); + } + node.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Blockchain.Singleton.Height))); + session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); + } + } + } } internal class TaskManagerMailbox : PriorityMailbox diff --git a/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs b/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs index 297b87f7f6..a5174d6cfe 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_RemoteNodeMailbox.cs @@ -44,6 +44,8 @@ public void RemoteNode_Test_IsHighPriority() //connectivity uut.IsHighPriority(Message.Create(MessageCommand.GetAddr, s)).Should().Be(false); uut.IsHighPriority(Message.Create(MessageCommand.Addr, s)).Should().Be(false); + uut.IsHighPriority(Message.Create(MessageCommand.Ping, s)).Should().Be(false); + uut.IsHighPriority(Message.Create(MessageCommand.Pong, s)).Should().Be(false); //synchronization uut.IsHighPriority(Message.Create(MessageCommand.GetHeaders, s)).Should().Be(false); @@ -108,6 +110,14 @@ public void ProtocolHandlerMailbox_Test_ShallDrop() msg = Message.Create(MessageCommand.Addr, s); uut.ShallDrop(msg, emptyQueue).Should().Be(false); uut.ShallDrop(msg, new object[] { msg }).Should().Be(false); + // Ping (no drop) + msg = Message.Create(MessageCommand.Ping, s); + uut.ShallDrop(msg, emptyQueue).Should().Be(false); + uut.ShallDrop(msg, new object[] { msg }).Should().Be(false); + // Pong (no drop) + msg = Message.Create(MessageCommand.Pong, s); + uut.ShallDrop(msg, emptyQueue).Should().Be(false); + uut.ShallDrop(msg, new object[] { msg }).Should().Be(false); //synchronization // GetHeaders (drop) From 77a0d7d5707f7fb52d6fa20693befa5eafd1219d Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 22 Jan 2021 16:55:34 +0800 Subject: [PATCH 04/91] fix --- tests/neo.UnitTests/Network/P2P/UT_Message.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/neo.UnitTests/Network/P2P/UT_Message.cs b/tests/neo.UnitTests/Network/P2P/UT_Message.cs index 3cb1b33a8e..c70e0b37fd 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_Message.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_Message.cs @@ -16,7 +16,7 @@ public class UT_Message public void Serialize_Deserialize() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Inv, payload); + var msg = Message.Create(MessageCommand.Ping, payload); var buffer = msg.ToArray(); var copy = buffer.AsSerializable(); var payloadCopy = (PingPayload)copy.Payload; @@ -34,7 +34,7 @@ public void Serialize_Deserialize() public void Serialize_Deserialize_ByteString() { var payload = PingPayload.Create(uint.MaxValue); - var msg = Message.Create(MessageCommand.Inv, payload); + var msg = Message.Create(MessageCommand.Ping, payload); var buffer = ByteString.CopyFrom(msg.ToArray()); var length = Message.TryDeserialize(buffer, out var copy); From f190df42253d846e12b13e63e1efb33701b31e54 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Sat, 23 Jan 2021 17:46:08 +0800 Subject: [PATCH 05/91] logic fix --- src/neo/Network/P2P/Payloads/HeadersPayload.cs | 2 ++ src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/Payloads/HeadersPayload.cs b/src/neo/Network/P2P/Payloads/HeadersPayload.cs index 2e95d4ebf4..9251a7f5a1 100644 --- a/src/neo/Network/P2P/Payloads/HeadersPayload.cs +++ b/src/neo/Network/P2P/Payloads/HeadersPayload.cs @@ -1,4 +1,5 @@ using Neo.IO; +using System; using System.IO; namespace Neo.Network.P2P.Payloads @@ -22,6 +23,7 @@ public static HeadersPayload Create(params Header[] headers) void ISerializable.Deserialize(BinaryReader reader) { Headers = reader.ReadSerializableArray
(MaxHeadersCount); + if (Headers.Length == 0) throw new FormatException(); } void ISerializable.Serialize(BinaryWriter writer) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 5bf07520c9..ba28ccf4c5 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -294,7 +294,6 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) private void OnHeadersMessageReceived(HeadersPayload payload) { - if (payload.Headers.Length == 0) return; system.Blockchain.Tell(payload.Headers); } From 87070ba76b03be7dc904907724362c7a30dcc3b7 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Mon, 25 Jan 2021 15:24:34 +0800 Subject: [PATCH 06/91] Update TaskManager/TaskSession --- src/neo/Ledger/Blockchain.cs | 1 + src/neo/Network/P2P/MessageCommand.cs | 7 +- .../P2P/Payloads/GetBlockByIndexPayload.cs | 16 +- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 46 ++-- src/neo/Network/P2P/TaskManager.cs | 218 +++++++----------- src/neo/Network/P2P/TaskSession.cs | 19 +- 6 files changed, 131 insertions(+), 176 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 4e5fbab19b..ffc7f5f15e 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -74,6 +74,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); public uint HeaderHeight => NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); + public UInt256 CurrentHeaderHash => NativeContract.Ledger.CurrentHeaderHash(currentSnapshot); private static Blockchain singleton; public static Blockchain Singleton diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index 04856c4a48..d04bdb7a3e 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -20,7 +20,7 @@ public enum MessageCommand : byte Pong = 0x19, //synchronization - [ReflectionCache(typeof(GetBlockByIndexPayload))] + [ReflectionCache(typeof(GetBlocksPayload))] GetHeaders = 0x20, [ReflectionCache(typeof(HeadersPayload))] Headers = 0x21, @@ -31,8 +31,9 @@ public enum MessageCommand : byte Inv = 0x27, [ReflectionCache(typeof(InvPayload))] GetData = 0x28, - [ReflectionCache(typeof(GetBlockByIndexPayload))] - GetBlockByIndex = 0x29, + [ReflectionCache(typeof(GetBlockDataPayload))] + GetBlockData = 0x29, + [ReflectionCache(typeof(InvPayload))] NotFound = 0x2a, [ReflectionCache(typeof(Transaction))] Transaction = 0x2b, diff --git a/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs b/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs index c07a8a8c0a..9ccd534f34 100644 --- a/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs +++ b/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs @@ -4,16 +4,17 @@ namespace Neo.Network.P2P.Payloads { - public class GetBlockByIndexPayload : ISerializable + public class GetBlockDataPayload : ISerializable { + private const ushort MaxBlocksCount = 500; public uint IndexStart; - public short Count; + public ushort Count; - public int Size => sizeof(uint) + sizeof(short); + public int Size => sizeof(uint) + sizeof(ushort); - public static GetBlockByIndexPayload Create(uint index_start, short count = -1) + public static GetBlockDataPayload Create(uint index_start, ushort count) { - return new GetBlockByIndexPayload + return new GetBlockDataPayload { IndexStart = index_start, Count = count @@ -23,9 +24,8 @@ public static GetBlockByIndexPayload Create(uint index_start, short count = -1) void ISerializable.Deserialize(BinaryReader reader) { IndexStart = reader.ReadUInt32(); - Count = reader.ReadInt16(); - if (Count < -1 || Count == 0 || Count > HeadersPayload.MaxHeadersCount) - throw new FormatException(); + Count = reader.ReadUInt16(); + if (Count == 0 || Count > MaxBlocksCount) throw new FormatException(); } void ISerializable.Serialize(BinaryWriter writer) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index ba28ccf4c5..c80c036b19 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -81,14 +81,14 @@ private void OnMessage(Message msg) case MessageCommand.GetBlocks: OnGetBlocksMessageReceived((GetBlocksPayload)msg.Payload); break; - case MessageCommand.GetBlockByIndex: - OnGetBlockByIndexMessageReceived((GetBlockByIndexPayload)msg.Payload); + case MessageCommand.GetBlockData: + OnGetBlockDataMessageReceived((GetBlockDataPayload)msg.Payload); break; case MessageCommand.GetData: OnGetDataMessageReceived((InvPayload)msg.Payload); break; case MessageCommand.GetHeaders: - OnGetHeadersMessageReceived((GetBlockByIndexPayload)msg.Payload); + OnGetHeadersMessageReceived((GetBlocksPayload)msg.Payload); break; case MessageCommand.Headers: OnHeadersMessageReceived((HeadersPayload)msg.Payload); @@ -194,10 +194,9 @@ private void OnGetBlocksMessageReceived(GetBlocksPayload payload) EnqueueMessage(Message.Create(MessageCommand.Inv, InvPayload.Create(InventoryType.Block, hashes.ToArray()))); } - private void OnGetBlockByIndexMessageReceived(GetBlockByIndexPayload payload) + private void OnGetBlockDataMessageReceived(GetBlockDataPayload payload) { - uint count = payload.Count == -1 ? InvPayload.MaxHashesCount : Math.Min((uint)payload.Count, InvPayload.MaxHashesCount); - for (uint i = payload.IndexStart, max = payload.IndexStart + count; i < max; i++) + for (uint i = payload.IndexStart, max = payload.IndexStart + payload.Count; i < max; i++) { Block block = NativeContract.Ledger.GetBlock(Blockchain.Singleton.View, i); if (block == null) @@ -269,27 +268,28 @@ private void OnGetDataMessageReceived(InvPayload payload) /// /// Will be triggered when a MessageCommand.GetHeaders message is received. - /// Tell the specified number of blocks' headers starting with the requested IndexStart to RemoteNode actor. + /// Tell the specified number of blocks' headers starting with the requested HashStart to RemoteNode actor. /// A limit set by HeadersPayload.MaxHeadersCount is also applied to the number of requested Headers, namely payload.Count. /// - /// A GetBlocksPayload including start block index and number of blocks' headers requested. - private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) + /// A GetBlocksPayload including start block Hash and number of blocks' headers requested. + private void OnGetHeadersMessageReceived(GetBlocksPayload payload) { - uint index = payload.IndexStart; - if (index > Blockchain.Singleton.Height) return; - List
headers = new List
(); + UInt256 hash = payload.HashStart; + int count = payload.Count < 0 || payload.Count > HeadersPayload.MaxHeadersCount ? HeadersPayload.MaxHeadersCount : payload.Count; using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) { - uint count = payload.Count == -1 ? HeadersPayload.MaxHeadersCount : (uint)payload.Count; - for (uint i = 0; i < count; i++) + Block block = NativeContract.Ledger.GetBlock(snapshot, hash); + if (block == null) return; + List
headers = new List
(); + for (uint i = 1; i <= count; i++) { - var header = NativeContract.Ledger.GetHeader(snapshot, index + i); + var header = NativeContract.Ledger.GetHeader(snapshot, block.Index + i); if (header == null) break; headers.Add(header); } + if (headers.Count == 0) return; + EnqueueMessage(Message.Create(MessageCommand.Headers, HeadersPayload.Create(headers.ToArray()))); } - if (headers.Count == 0) return; - EnqueueMessage(Message.Create(MessageCommand.Headers, HeadersPayload.Create(headers.ToArray()))); } private void OnHeadersMessageReceived(HeadersPayload payload) @@ -303,10 +303,10 @@ private void OnInventoryReceived(IInventory inventory) if (inventory is Block block) { if (block.Index > Blockchain.Singleton.Height + InvPayload.MaxHashesCount) return; - UpdateLastBlockIndex(block.Index, false); + UpdateLastBlockIndex(block.Index); } knownHashes.Add(inventory.Hash); - system.TaskManager.Tell(inventory); + system.TaskManager.Tell(new TaskManager.TaskCompleted { Hash = inventory.Hash }); system.Blockchain.Tell(inventory); } @@ -339,13 +339,13 @@ private void OnMemPoolMessageReceived() private void OnPingMessageReceived(PingPayload payload) { - UpdateLastBlockIndex(payload.LastBlockIndex, true); + UpdateLastBlockIndex(payload.LastBlockIndex); EnqueueMessage(Message.Create(MessageCommand.Pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); } private void OnPongMessageReceived(PingPayload payload) { - UpdateLastBlockIndex(payload.LastBlockIndex, true); + UpdateLastBlockIndex(payload.LastBlockIndex); } private void OnVerackMessageReceived() @@ -391,12 +391,12 @@ private void RefreshPendingKnownHashes() } } - private void UpdateLastBlockIndex(uint lastBlockIndex, bool requestTasks) + private void UpdateLastBlockIndex(uint lastBlockIndex) { if (lastBlockIndex > LastBlockIndex) { LastBlockIndex = lastBlockIndex; - system.TaskManager.Tell(new TaskManager.Update { LastBlockIndex = LastBlockIndex, RequestTasks = requestTasks }); + system.TaskManager.Tell(new TaskManager.Update { LastBlockIndex = LastBlockIndex }); } } } diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 1bb8d4176a..4fbc17e2fa 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -17,8 +17,9 @@ namespace Neo.Network.P2P public class TaskManager : UntypedActor { internal class Register { public VersionPayload Version; } - internal class Update { public uint LastBlockIndex; public bool RequestTasks; } + internal class Update { public uint LastBlockIndex; } internal class NewTasks { public InvPayload Payload; } + public class TaskCompleted { public UInt256 Hash; } public class HeaderTaskCompleted { } public class RestartTasks { public InvPayload Payload; } private class Timer { } @@ -29,7 +30,6 @@ private class Timer { } private static readonly UInt256 MemPoolTaskHash = UInt256.Parse("0x0000000000000000000000000000000000000000000000000000000000000001"); private const int MaxConncurrentTasks = 3; - private const int MaxSyncTasksCount = 50; private const int PingCoolingOffPeriod = 60_000; // in ms. private readonly NeoSystem system; @@ -38,19 +38,14 @@ private class Timer { } /// private readonly HashSetCache knownHashes; private readonly Dictionary globalTasks = new Dictionary(); - private readonly Dictionary receivedBlockIndex = new Dictionary(); - private readonly HashSet failedSyncTasks = new HashSet(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); private bool HasHeaderTask => globalTasks.ContainsKey(HeaderTaskHash); - private uint lastTaskIndex = 0; public TaskManager(NeoSystem system) { this.system = system; this.knownHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); - this.lastTaskIndex = Blockchain.Singleton.Height; - Context.System.EventStream.Subscribe(Self, typeof(Blockchain.PersistCompleted)); Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } @@ -58,49 +53,9 @@ private void OnHeaderTaskCompleted() { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; - session.InvTasks.Remove(HeaderTaskHash); + session.Tasks.Remove(HeaderTaskHash); DecrementGlobalTask(HeaderTaskHash); - RequestTasks(true); - } - - private bool AssignSyncTask(uint index, TaskSession filterSession = null) - { - if (index <= Blockchain.Singleton.Height || sessions.Values.Any(p => p != filterSession && p.IndexTasks.ContainsKey(index))) - return true; - Random rand = new Random(); - KeyValuePair remoteNode = sessions.Where(p => p.Value != filterSession && p.Value.LastBlockIndex >= index) - .OrderBy(p => p.Value.IndexTasks.Count) - .ThenBy(s => rand.Next()) - .FirstOrDefault(); - if (remoteNode.Value == null) - { - failedSyncTasks.Add(index); - return false; - } - TaskSession session = remoteNode.Value; - session.IndexTasks.TryAdd(index, TimeProvider.Current.UtcNow); - remoteNode.Key.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(index, 1))); - failedSyncTasks.Remove(index); - return true; - } - - private void OnBlock(Block block) - { - var session = sessions.Values.FirstOrDefault(p => p.IndexTasks.ContainsKey(block.Index)); - if (session is null) return; - session.IndexTasks.Remove(block.Index); - receivedBlockIndex.TryAdd(block.Index, session); - RequestTasks(false); - } - - private void OnInvalidBlock(Block invalidBlock) - { - receivedBlockIndex.TryGetValue(invalidBlock.Index, out TaskSession session); - if (session is null) return; - session.InvalidBlockCount++; - session.IndexTasks.Remove(invalidBlock.Index); - receivedBlockIndex.Remove(invalidBlock.Index); - AssignSyncTask(invalidBlock.Index, session); + RequestTasks(session); } private void OnNewTasks(InvPayload payload) @@ -108,34 +63,37 @@ private void OnNewTasks(InvPayload payload) if (!sessions.TryGetValue(Sender, out TaskSession session)) return; // Do not accept payload of type InventoryType.TX if not synced on best known HeaderHeight - if (payload.Type == InventoryType.TX && Blockchain.Singleton.Height < sessions.Values.Max(p => p.LastBlockIndex)) + if (payload.Type == InventoryType.TX && Blockchain.Singleton.Height < Blockchain.Singleton.HeaderHeight) + { + RequestTasks(session); return; + } HashSet hashes = new HashSet(payload.Hashes); // Remove all previously processed knownHashes from the list that is being requested hashes.Remove(knownHashes); + // Add to AvailableTasks the ones, of type InventoryType.Block, that are global (already under process by other sessions) + if (payload.Type == InventoryType.Block) + session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p))); // Remove those that are already in process by other sessions hashes.Remove(globalTasks); if (hashes.Count == 0) + { + RequestTasks(session); return; + } // Update globalTasks with the ones that will be requested within this current session foreach (UInt256 hash in hashes) { IncrementGlobalTask(hash); - session.InvTasks[hash] = TimeProvider.Current.UtcNow; + session.Tasks[hash] = DateTime.UtcNow; } foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, hashes.ToArray())) Sender.Tell(Message.Create(MessageCommand.GetData, group)); } - private void OnPersistCompleted(Block block) - { - receivedBlockIndex.Remove(block.Index); - RequestTasks(false); - } - protected override void OnReceive(object message) { switch (message) @@ -149,25 +107,15 @@ protected override void OnReceive(object message) case NewTasks tasks: OnNewTasks(tasks.Payload); break; + case TaskCompleted completed: + OnTaskCompleted(completed.Hash); + break; case HeaderTaskCompleted _: OnHeaderTaskCompleted(); break; case RestartTasks restart: OnRestartTasks(restart.Payload); break; - case Block block: - OnBlock(block); - break; - case IInventory inventory: - OnTaskCompleted(inventory.Hash); - break; - case Blockchain.PersistCompleted pc: - OnPersistCompleted(pc.Block); - break; - case Blockchain.RelayResult rr: - if (rr.Inventory is Block invalidBlock && rr.Result == VerifyResult.Invalid) - OnInvalidBlock(invalidBlock); - break; case Timer _: OnTimer(); break; @@ -180,11 +128,11 @@ protected override void OnReceive(object message) private void OnRegister(VersionPayload version) { Context.Watch(Sender); - TaskSession session = new TaskSession(version); + TaskSession session = new TaskSession(Sender, version); if (session.IsFullNode) - session.InvTasks.TryAdd(MemPoolTaskHash, TimeProvider.Current.UtcNow); - sessions.TryAdd(Sender, session); - RequestTasks(true); + session.AvailableTasks.Add(TaskManager.MemPoolTaskHash); + sessions.Add(Sender, session); + RequestTasks(session); } private void OnUpdate(Update update) @@ -192,8 +140,6 @@ private void OnUpdate(Update update) if (!sessions.TryGetValue(Sender, out TaskSession session)) return; session.LastBlockIndex = update.LastBlockIndex; - session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); - if (update.RequestTasks) RequestTasks(true); } private void OnRestartTasks(InvPayload payload) @@ -209,8 +155,13 @@ private void OnTaskCompleted(UInt256 hash) { knownHashes.Add(hash); globalTasks.Remove(hash); + foreach (TaskSession ms in sessions.Values) + ms.AvailableTasks.Remove(hash); if (sessions.TryGetValue(Sender, out TaskSession session)) - session.InvTasks.Remove(hash); + { + session.Tasks.Remove(hash); + RequestTasks(session); + } } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -244,10 +195,7 @@ private void OnTerminated(IActorRef actor) { if (!sessions.TryGetValue(actor, out TaskSession session)) return; - foreach (uint index in session.IndexTasks.Keys) - AssignSyncTask(index, session); - - foreach (UInt256 hash in session.InvTasks.Keys) + foreach (UInt256 hash in session.Tasks.Keys) DecrementGlobalTask(hash); sessions.Remove(actor); } @@ -255,27 +203,14 @@ private void OnTerminated(IActorRef actor) private void OnTimer() { foreach (TaskSession session in sessions.Values) - { - foreach (KeyValuePair kvp in session.IndexTasks) - { - if (TimeProvider.Current.UtcNow - kvp.Value > TaskTimeout) - { - session.IndexTasks.Remove(kvp.Key); - session.TimeoutTimes++; - AssignSyncTask(kvp.Key, session); - } - } - - foreach (var task in session.InvTasks.ToArray()) - { + foreach (var task in session.Tasks.ToArray()) if (TimeProvider.Current.UtcNow - task.Value > TaskTimeout) { - if (session.InvTasks.Remove(task.Key)) + if (session.Tasks.Remove(task.Key)) DecrementGlobalTask(task.Key); } - } - } - RequestTasks(true); + foreach (TaskSession session in sessions.Values) + RequestTasks(session); } protected override void PostStop() @@ -289,56 +224,68 @@ public static Props Props(NeoSystem system) return Akka.Actor.Props.Create(() => new TaskManager(system)).WithMailbox("task-manager-mailbox"); } - private void RequestTasks(bool sendPing) + private void RequestTasks(TaskSession session) { - if (sessions.Count() == 0) return; - - if (sendPing) SendPingMessage(); - - while (failedSyncTasks.Count() > 0) + if (session.HasTask) return; + // If there are pending tasks of InventoryType.Block we should process them + if (session.AvailableTasks.Count > 0) { - var failedTask = failedSyncTasks.First(); - if (failedTask <= Blockchain.Singleton.Height) + session.AvailableTasks.Remove(knownHashes); + // Search any similar hash that is on Singleton's knowledge, which means, on the way or already processed + using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) + session.AvailableTasks.RemoveWhere(p => NativeContract.Ledger.ContainsBlock(snapshot, p)); + HashSet hashes = new HashSet(session.AvailableTasks); + hashes.Remove(MemPoolTaskHash); + if (hashes.Count > 0) { - failedSyncTasks.Remove(failedTask); - continue; + foreach (UInt256 hash in hashes.ToArray()) + { + if (!IncrementGlobalTask(hash)) + hashes.Remove(hash); + } + session.AvailableTasks.Remove(hashes); + foreach (UInt256 hash in hashes) + session.Tasks[hash] = DateTime.UtcNow; + foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray())) + session.RemoteNode.Tell(Message.Create(MessageCommand.GetData, group)); + return; } - if (!AssignSyncTask(failedTask)) return; } - int taskCounts = sessions.Values.Sum(p => p.IndexTasks.Count); - var highestBlockIndex = sessions.Values.Max(p => p.LastBlockIndex); - for (; taskCounts < MaxSyncTasksCount; taskCounts++) + // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers + // If not HeaderTask pending to be processed it should ask for more Blocks + if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && Blockchain.Singleton.HeaderHeight < session.LastBlockIndex) { - if (lastTaskIndex >= highestBlockIndex || lastTaskIndex >= Blockchain.Singleton.Height + InvPayload.MaxHashesCount) break; - if (!AssignSyncTask(++lastTaskIndex)) break; + session.Tasks[HeaderTaskHash] = DateTime.UtcNow; + IncrementGlobalTask(HeaderTaskHash); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); } - } - - private void SendPingMessage() - { - TrimmedBlock block; - using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) - { - block = NativeContract.Ledger.GetTrimmedBlock(snapshot, NativeContract.Ledger.CurrentHash(snapshot)); - } - - foreach (KeyValuePair item in sessions) + else if (Blockchain.Singleton.Height < session.LastBlockIndex) { - var node = item.Key; - var session = item.Value; - - if (session.ExpireTime < TimeProvider.Current.UtcNow || - (block.Index >= session.LastBlockIndex && - TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= block.Timestamp)) + UInt256 hash = Blockchain.Singleton.CurrentBlockHash; + using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) { - if (session.InvTasks.Remove(MemPoolTaskHash)) + for (uint i = Blockchain.Singleton.Height + 1; i <= Blockchain.Singleton.HeaderHeight; i++) { - node.Tell(Message.Create(MessageCommand.Mempool)); + hash = NativeContract.Ledger.GetBlockHash(snapshot, i); + if (!globalTasks.ContainsKey(hash)) + { + hash = NativeContract.Ledger.GetBlockHash(snapshot, i - 1); + break; + } } - node.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Blockchain.Singleton.Height))); - session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); } + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); + } + else if (Blockchain.Singleton.HeaderHeight >= session.LastBlockIndex + && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(Blockchain.Singleton.GetSnapshot(), Blockchain.Singleton.CurrentHeaderHash)?.Timestamp) + { + if (session.AvailableTasks.Remove(MemPoolTaskHash)) + { + session.RemoteNode.Tell(Message.Create(MessageCommand.Mempool)); + } + + session.RemoteNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Blockchain.Singleton.Height))); } } } @@ -355,6 +302,7 @@ internal protected override bool IsHighPriority(object message) switch (message) { case TaskManager.Register _: + case TaskManager.Update _: case TaskManager.RestartTasks _: return true; case TaskManager.NewTasks tasks: diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index b8a78bd5f6..c4ca9c6004 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -1,3 +1,4 @@ +using Akka.Actor; using Neo.Network.P2P.Capabilities; using Neo.Network.P2P.Payloads; using System; @@ -8,20 +9,24 @@ namespace Neo.Network.P2P { internal class TaskSession { - public readonly Dictionary InvTasks = new Dictionary(); - public readonly Dictionary IndexTasks = new Dictionary(); + public readonly IActorRef RemoteNode; + public readonly VersionPayload Version; + public readonly Dictionary Tasks = new Dictionary(); + public readonly HashSet AvailableTasks = new HashSet(); + public bool HasTask => Tasks.Count > 0; + public uint StartHeight { get; } public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } - public uint TimeoutTimes = 0; - public uint InvalidBlockCount = 0; - public DateTime ExpireTime = DateTime.MinValue; - public TaskSession(VersionPayload version) + public TaskSession(IActorRef node, VersionPayload version) { var fullNode = version.Capabilities.OfType().FirstOrDefault(); this.IsFullNode = fullNode != null; - this.LastBlockIndex = fullNode?.StartHeight ?? 0; + this.RemoteNode = node; + this.Version = version; + this.StartHeight = fullNode?.StartHeight ?? 0; + this.LastBlockIndex = this.StartHeight; } } } From 02be1c76d058f3f7ba0e84f463a8115310e94cae Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Mon, 25 Jan 2021 16:25:37 +0800 Subject: [PATCH 07/91] fix ut --- .../P2P/Payloads/UT_GetBlockByIndexPayload.cs | 41 ------------------- .../P2P/Payloads/UT_GetBlockDataPayload.cs | 38 +++++++++++++++++ .../Network/P2P/UT_TaskSession.cs | 25 +++++------ 3 files changed, 48 insertions(+), 56 deletions(-) delete mode 100644 tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs create mode 100644 tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs diff --git a/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs b/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs deleted file mode 100644 index 76ed1df9f9..0000000000 --- a/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs +++ /dev/null @@ -1,41 +0,0 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.IO; -using Neo.Network.P2P.Payloads; -using System; - -namespace Neo.UnitTests.Network.P2P.Payloads -{ - [TestClass] - public class UT_GetBlockByIndexPayload - { - [TestMethod] - public void Size_Get() - { - var test = new GetBlockByIndexPayload() { Count = 5, IndexStart = 5 }; - test.Size.Should().Be(6); - - test = GetBlockByIndexPayload.Create(1, short.MaxValue); - test.Size.Should().Be(6); - } - - [TestMethod] - public void DeserializeAndSerialize() - { - var test = new GetBlockByIndexPayload() { Count = -1, IndexStart = int.MaxValue }; - var clone = test.ToArray().AsSerializable(); - - Assert.AreEqual(test.Count, clone.Count); - Assert.AreEqual(test.IndexStart, clone.IndexStart); - - test = new GetBlockByIndexPayload() { Count = -2, IndexStart = int.MaxValue }; - Assert.ThrowsException(() => test.ToArray().AsSerializable()); - - test = new GetBlockByIndexPayload() { Count = 0, IndexStart = int.MaxValue }; - Assert.ThrowsException(() => test.ToArray().AsSerializable()); - - test = new GetBlockByIndexPayload() { Count = HeadersPayload.MaxHeadersCount + 1, IndexStart = int.MaxValue }; - Assert.ThrowsException(() => test.ToArray().AsSerializable()); - } - } -} diff --git a/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs b/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs new file mode 100644 index 0000000000..b6e6df2839 --- /dev/null +++ b/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs @@ -0,0 +1,38 @@ +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO; +using Neo.Network.P2P.Payloads; +using System; + +namespace Neo.UnitTests.Network.P2P.Payloads +{ + [TestClass] + public class UT_GetBlockDataPayload + { + [TestMethod] + public void Size_Get() + { + var test = new GetBlockDataPayload() { Count = 5, IndexStart = 5 }; + test.Size.Should().Be(6); + + test = GetBlockDataPayload.Create(1, ushort.MaxValue); + test.Size.Should().Be(6); + } + + [TestMethod] + public void DeserializeAndSerialize() + { + var test = new GetBlockDataPayload() { Count = 1, IndexStart = int.MaxValue }; + var clone = test.ToArray().AsSerializable(); + + Assert.AreEqual(test.Count, clone.Count); + Assert.AreEqual(test.IndexStart, clone.IndexStart); + + test = new GetBlockDataPayload() { Count = 0, IndexStart = int.MaxValue }; + Assert.ThrowsException(() => test.ToArray().AsSerializable()); + + test = new GetBlockDataPayload() { Count = 501, IndexStart = int.MaxValue }; + Assert.ThrowsException(() => test.ToArray().AsSerializable()); + } + } +} diff --git a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs index 6ae403a946..b181eaf352 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs @@ -2,7 +2,6 @@ using Neo.Network.P2P; using Neo.Network.P2P.Capabilities; using Neo.Network.P2P.Payloads; -using System; using Xunit.Sdk; namespace Neo.UnitTests.Network.P2P @@ -13,25 +12,21 @@ public class UT_TaskSession [TestMethod] public void CreateTest() { - Assert.ThrowsException(() => new TaskSession(null)); + var ses = new TaskSession(null, new VersionPayload() { Capabilities = new NodeCapability[] { new FullNodeCapability(123) } }); - var ses = new TaskSession(new VersionPayload() { Capabilities = new NodeCapability[] { new FullNodeCapability(123) } }); - - Assert.IsTrue(ses.IsFullNode); + Assert.IsNull(ses.RemoteNode); + Assert.IsFalse(ses.HasTask); + Assert.AreEqual((uint)123, ses.StartHeight); Assert.AreEqual((uint)123, ses.LastBlockIndex); - Assert.AreEqual(0, ses.IndexTasks.Count); - Assert.AreEqual(0, ses.InvTasks.Count); - Assert.AreEqual((uint)0, ses.TimeoutTimes); - Assert.AreEqual((uint)0, ses.InvalidBlockCount); + Assert.IsTrue(ses.IsFullNode); - ses = new TaskSession(new VersionPayload() { Capabilities = new NodeCapability[0] }); + ses = new TaskSession(null, new VersionPayload() { Capabilities = new NodeCapability[0] }); - Assert.IsFalse(ses.IsFullNode); + Assert.IsNull(ses.RemoteNode); + Assert.IsFalse(ses.HasTask); + Assert.AreEqual((uint)0, ses.StartHeight); Assert.AreEqual((uint)0, ses.LastBlockIndex); - Assert.AreEqual(0, ses.IndexTasks.Count); - Assert.AreEqual(0, ses.InvTasks.Count); - Assert.AreEqual((uint)0, ses.TimeoutTimes); - Assert.AreEqual((uint)0, ses.InvalidBlockCount); + Assert.IsFalse(ses.IsFullNode); } } } From 84b562c48d11f035111492416159e399d17b09a3 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 25 Jan 2021 10:09:59 +0100 Subject: [PATCH 08/91] Array.Empty --- src/neo/Network/P2P/Payloads/Header.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/Payloads/Header.cs b/src/neo/Network/P2P/Payloads/Header.cs index ffabe7a172..6d76298d47 100644 --- a/src/neo/Network/P2P/Payloads/Header.cs +++ b/src/neo/Network/P2P/Payloads/Header.cs @@ -48,7 +48,7 @@ public TrimmedBlock Trim() Index = Index, NextConsensus = NextConsensus, Witness = Witness, - Hashes = new UInt256[0] + Hashes = Array.Empty() }; } } From a7320acb1d570124fc6729619f3bab2469c854e7 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Mon, 25 Jan 2021 18:25:40 +0800 Subject: [PATCH 09/91] kick line --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 4fbc17e2fa..380e07a62b 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -26,8 +26,8 @@ private class Timer { } private static readonly TimeSpan TimerInterval = TimeSpan.FromSeconds(30); private static readonly TimeSpan TaskTimeout = TimeSpan.FromMinutes(1); - private static readonly UInt256 HeaderTaskHash = UInt256.Zero; private static readonly UInt256 MemPoolTaskHash = UInt256.Parse("0x0000000000000000000000000000000000000000000000000000000000000001"); + private static readonly UInt256 HeaderTaskHash = UInt256.Zero; private const int MaxConncurrentTasks = 3; private const int PingCoolingOffPeriod = 60_000; // in ms. From ffc236fc2cf300f460a64d31a753e862644b8088 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 16:34:11 +0800 Subject: [PATCH 10/91] add header cache --- src/neo/IO/Caching/HeaderCache.cs | 65 +++++++++++++++++++ src/neo/Ledger/Blockchain.cs | 40 +++++++++--- .../SmartContract/Native/LedgerContract.cs | 4 ++ 3 files changed, 100 insertions(+), 9 deletions(-) create mode 100644 src/neo/IO/Caching/HeaderCache.cs diff --git a/src/neo/IO/Caching/HeaderCache.cs b/src/neo/IO/Caching/HeaderCache.cs new file mode 100644 index 0000000000..63e4263fe6 --- /dev/null +++ b/src/neo/IO/Caching/HeaderCache.cs @@ -0,0 +1,65 @@ +using Neo.Network.P2P.Payloads; +using System; + +namespace Neo.IO.Caching +{ + internal class HeaderCache + { + private int max_capacity; + private int startPos = -1; + private int endPos = -1; + private uint startIndex; + private uint endIndex; + private Header[] headers = null; + + public HeaderCache(int max_capacity) + { + if (max_capacity <= 0) throw new ArgumentException("illegal max_capacity"); + this.max_capacity = max_capacity; + headers = new Header[max_capacity]; + } + + public bool Added => startPos != -1; + + public void Add (Header header) + { + if (!Added) + { + startIndex = header.Index; + endIndex = header.Index; + startPos = 0; + endPos = 0; + headers[0] = header; + return; + } + else + { + if (header.Index != headers[endPos].Index + 1) throw new ArgumentException("illegal header"); + endPos = (endPos + 1) % max_capacity; + endIndex++; + } + if (endPos == startPos) + { + startPos = (startPos + 1) % max_capacity; + startIndex++; + } + headers[endPos] = header; + } + + public Header At(uint index) + { + if (startPos == -1 || index < startIndex || index > endIndex) return null; + return headers[(startPos + index - startIndex) % max_capacity]; + } + + public uint HeaderHeight() + { + return endIndex; + } + + public Header CurrentHeader() + { + return endPos == -1 ? null : headers[endPos]; + } + } +} diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index ffc7f5f15e..031e63ae27 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -65,6 +65,8 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private readonly ConcurrentDictionary block_cache = new ConcurrentDictionary(); private readonly Dictionary block_cache_unverified = new Dictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); + private readonly HeaderCache recentHeaders = new HeaderCache(10000); + private SnapshotCache currentSnapshot; private ImmutableHashSet extensibleWitnessWhiteList; @@ -72,9 +74,9 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); - public uint HeaderHeight => NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot); + public uint HeaderHeight => recentHeaders.Added ? recentHeaders.HeaderHeight() : NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); - public UInt256 CurrentHeaderHash => NativeContract.Ledger.CurrentHeaderHash(currentSnapshot); + public UInt256 CurrentHeaderHash => recentHeaders.Added ? recentHeaders.CurrentHeader().Hash : NativeContract.Ledger.CurrentHeaderHash(currentSnapshot); private static Blockchain singleton; public static Blockchain Singleton @@ -230,7 +232,7 @@ private VerifyResult OnNewBlock(Block block) } else { - if (!block.Hash.Equals(NativeContract.Ledger.GetBlockHash(currentSnapshot, block.Index))) + if (!block.Hash.Equals(GetBlockHash(block.Index))) return VerifyResult.Invalid; } block_cache.TryAdd(block.Hash, block); @@ -241,8 +243,8 @@ private VerifyResult OnNewBlock(Block block) while (true) { blocksToPersistList.Add(block_persist); - if (block_persist.Index + 1 >= NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot)) break; - UInt256 hash = NativeContract.Ledger.CurrentHeaderHash(currentSnapshot); + if (block_persist.Index + 1 > HeaderHeight) break; + UInt256 hash = recentHeaders.At(block_persist.Index + 1).Hash; if (!block_cache.TryGetValue(hash, out block_persist)) break; } @@ -259,7 +261,7 @@ private VerifyResult OnNewBlock(Block block) // Empirically calibrated for relaying the most recent 2 blocks persisted with 15s network // Increase in the rate of 1 block per second in configurations with faster blocks - if (blockToPersist.Index + 100 >= NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot)) + if (blockToPersist.Index + 99 >= HeaderHeight) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = blockToPersist }); } if (block_cache_unverified.TryGetValue(Height + 1, out var unverifiedBlocks)) @@ -273,8 +275,20 @@ private VerifyResult OnNewBlock(Block block) } else { - if (block.Index + 100 >= NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot)) + if (block.Index + 99 >= HeaderHeight) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = block }); + if (block.Index == HeaderHeight + 1) + { + Header header = block.Header; + recentHeaders.Add(header); + using (SnapshotCache snapshot = GetSnapshot()) + { + NativeContract.Ledger.SaveHeader(snapshot, header); + NativeContract.Ledger.SetCurrentHeader(snapshot, header.Hash, header.Index); + snapshot.Commit(); + } + UpdateCurrentSnapshot(); + } } return VerifyResult.Succeed; } @@ -285,9 +299,10 @@ private void OnNewHeaders(Header[] headers) { foreach (Header header in headers) { - if (header.Index - 1 >= HeaderHeight) break; - if (header.Index < HeaderHeight) continue; + if (header.Index > HeaderHeight + 1) break; + if (header.Index < HeaderHeight + 1) continue; if (!header.Verify(snapshot)) break; + recentHeaders.Add(header); NativeContract.Ledger.SaveHeader(snapshot, header); NativeContract.Ledger.SetCurrentHeader(snapshot, header.Hash, header.Index); } @@ -362,6 +377,7 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { + if (block.Index == 0) NativeContract.Ledger.SetCurrentHeader(snapshot, block.Hash, block.Index); List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) { @@ -450,6 +466,12 @@ private void SendRelayResult(IInventory inventory, VerifyResult result) Context.System.EventStream.Publish(rr); } + private UInt256 GetBlockHash(uint index) + { + UInt256 hash = recentHeaders.At(index)?.Hash; + return hash != null ? hash : NativeContract.Ledger.GetBlockHash(currentSnapshot, index); + } + private void UpdateCurrentSnapshot() { Interlocked.Exchange(ref currentSnapshot, GetSnapshot())?.Dispose(); diff --git a/src/neo/SmartContract/Native/LedgerContract.cs b/src/neo/SmartContract/Native/LedgerContract.cs index 834f6f8a04..b67b785c6a 100644 --- a/src/neo/SmartContract/Native/LedgerContract.cs +++ b/src/neo/SmartContract/Native/LedgerContract.cs @@ -40,6 +40,10 @@ internal override void PostPersist(ApplicationEngine engine) HashIndexState state = engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_CurrentBlock), () => new StorageItem(new HashIndexState())).GetInteroperable(); state.Hash = engine.PersistingBlock.Hash; state.Index = engine.PersistingBlock.Index; + if (CurrentHeaderIndex(engine.Snapshot) < engine.PersistingBlock.Index) + { + SetCurrentHeader(engine.Snapshot, engine.PersistingBlock.Hash, engine.PersistingBlock.Index); + } } internal bool Initialized(DataCache snapshot) From 90c8d95add1ec5bbca433e2d984b902a4c0e8582 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 16:41:49 +0800 Subject: [PATCH 11/91] format --- src/neo/IO/Caching/HeaderCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/IO/Caching/HeaderCache.cs b/src/neo/IO/Caching/HeaderCache.cs index 63e4263fe6..aee01c54a3 100644 --- a/src/neo/IO/Caching/HeaderCache.cs +++ b/src/neo/IO/Caching/HeaderCache.cs @@ -21,7 +21,7 @@ public HeaderCache(int max_capacity) public bool Added => startPos != -1; - public void Add (Header header) + public void Add(Header header) { if (!Added) { From a1f56fc533774a0dd6e78698a4cd9f1cd992ba39 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 17:48:25 +0800 Subject: [PATCH 12/91] fix --- src/neo/Ledger/Blockchain.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 031e63ae27..932444667b 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -377,7 +377,15 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { - if (block.Index == 0) NativeContract.Ledger.SetCurrentHeader(snapshot, block.Hash, block.Index); + if (block.Index == 0) + { + NativeContract.Ledger.SetCurrentHeader(snapshot, block.Hash, block.Index); + recentHeaders.Add(block.Header); + } + else if (block.Index == HeaderHeight + 1) + { + recentHeaders.Add(block.Header); + } List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) { From 4206e004af82c8f19c9c7625c1e3952099a6129c Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 17:58:27 +0800 Subject: [PATCH 13/91] rename parameter --- src/neo/IO/Caching/HeaderCache.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neo/IO/Caching/HeaderCache.cs b/src/neo/IO/Caching/HeaderCache.cs index aee01c54a3..e0ea8d03d0 100644 --- a/src/neo/IO/Caching/HeaderCache.cs +++ b/src/neo/IO/Caching/HeaderCache.cs @@ -46,10 +46,10 @@ public void Add(Header header) headers[endPos] = header; } - public Header At(uint index) + public Header At(uint height) { - if (startPos == -1 || index < startIndex || index > endIndex) return null; - return headers[(startPos + index - startIndex) % max_capacity]; + if (startPos == -1 || height < startIndex || height > endIndex) return null; + return headers[(startPos + height - startIndex) % max_capacity]; } public uint HeaderHeight() From 0a85bdc077b383c8b7afeec6422cdfdaafe4478a Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 18:11:58 +0800 Subject: [PATCH 14/91] fix --- src/neo/SmartContract/Native/LedgerContract.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/neo/SmartContract/Native/LedgerContract.cs b/src/neo/SmartContract/Native/LedgerContract.cs index b67b785c6a..7a260bb141 100644 --- a/src/neo/SmartContract/Native/LedgerContract.cs +++ b/src/neo/SmartContract/Native/LedgerContract.cs @@ -23,7 +23,7 @@ internal LedgerContract() internal override void OnPersist(ApplicationEngine engine) { - engine.Snapshot.Add(CreateStorageKey(Prefix_BlockHash).AddBigEndian(engine.PersistingBlock.Index), new StorageItem(engine.PersistingBlock.Hash.ToArray(), true)); + engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_BlockHash).AddBigEndian(engine.PersistingBlock.Index), () => new StorageItem(engine.PersistingBlock.Hash.ToArray(), true)); engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_Block).Add(engine.PersistingBlock.Hash), () => new StorageItem(Trim(engine.PersistingBlock).ToArray(), true)); foreach (Transaction tx in engine.PersistingBlock.Transactions) { @@ -176,6 +176,7 @@ public uint CurrentHeaderIndex(DataCache snapshot) public void SaveHeader(DataCache snapshot, Header header) { + snapshot.Add(CreateStorageKey(Prefix_BlockHash).AddBigEndian(header.Index), new StorageItem(header.Hash.ToArray(), true)); snapshot.Add(CreateStorageKey(Prefix_Block).Add(header.Hash), new StorageItem(header.Trim().ToArray(), true)); } From 82516be07ab2b831623c1b06bbd270be494929dd Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 19:24:34 +0800 Subject: [PATCH 15/91] header only in cache --- src/neo/IO/Caching/HeaderCache.cs | 76 ++++++++++++++----- src/neo/Ledger/Blockchain.cs | 24 +----- ...IndexPayload.cs => GetBlockDataPayload.cs} | 0 src/neo/Network/P2P/TaskManager.cs | 4 +- .../SmartContract/Native/LedgerContract.cs | 32 +------- 5 files changed, 63 insertions(+), 73 deletions(-) rename src/neo/Network/P2P/Payloads/{GetBlockByIndexPayload.cs => GetBlockDataPayload.cs} (100%) diff --git a/src/neo/IO/Caching/HeaderCache.cs b/src/neo/IO/Caching/HeaderCache.cs index e0ea8d03d0..163105b66b 100644 --- a/src/neo/IO/Caching/HeaderCache.cs +++ b/src/neo/IO/Caching/HeaderCache.cs @@ -1,5 +1,6 @@ using Neo.Network.P2P.Payloads; using System; +using System.Threading; namespace Neo.IO.Caching { @@ -11,6 +12,7 @@ internal class HeaderCache private uint startIndex; private uint endIndex; private Header[] headers = null; + protected readonly ReaderWriterLockSlim RwSyncRootLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); public HeaderCache(int max_capacity) { @@ -23,43 +25,75 @@ public HeaderCache(int max_capacity) public void Add(Header header) { - if (!Added) + try { - startIndex = header.Index; - endIndex = header.Index; - startPos = 0; - endPos = 0; - headers[0] = header; - return; + RwSyncRootLock.EnterWriteLock(); + if (!Added) + { + startIndex = header.Index; + endIndex = header.Index; + startPos = 0; + endPos = 0; + headers[0] = header; + return; + } + else + { + if (header.Index != headers[endPos].Index + 1) throw new ArgumentException("illegal header"); + endPos = (endPos + 1) % max_capacity; + endIndex++; + } + if (endPos == startPos) + { + startPos = (startPos + 1) % max_capacity; + startIndex++; + } + headers[endPos] = header; } - else + finally { - if (header.Index != headers[endPos].Index + 1) throw new ArgumentException("illegal header"); - endPos = (endPos + 1) % max_capacity; - endIndex++; + RwSyncRootLock.ExitWriteLock(); } - if (endPos == startPos) - { - startPos = (startPos + 1) % max_capacity; - startIndex++; - } - headers[endPos] = header; } public Header At(uint height) { - if (startPos == -1 || height < startIndex || height > endIndex) return null; - return headers[(startPos + height - startIndex) % max_capacity]; + try + { + RwSyncRootLock.EnterReadLock(); + if (startPos == -1 || height < startIndex || height > endIndex) return null; + return headers[(startPos + height - startIndex) % max_capacity]; + } + finally + { + RwSyncRootLock.ExitReadLock(); + } } public uint HeaderHeight() { - return endIndex; + try + { + RwSyncRootLock.EnterReadLock(); + return endIndex; + } + finally + { + RwSyncRootLock.ExitReadLock(); + } } public Header CurrentHeader() { - return endPos == -1 ? null : headers[endPos]; + try + { + RwSyncRootLock.EnterReadLock(); + return endPos == -1 ? null : headers[endPos]; + } + finally + { + RwSyncRootLock.ExitReadLock(); + } } } } diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 932444667b..3f2f3355a9 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -74,9 +74,9 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); - public uint HeaderHeight => recentHeaders.Added ? recentHeaders.HeaderHeight() : NativeContract.Ledger.CurrentHeaderIndex(currentSnapshot); + public uint HeaderHeight => recentHeaders.Added ? recentHeaders.HeaderHeight() : NativeContract.Ledger.CurrentIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); - public UInt256 CurrentHeaderHash => recentHeaders.Added ? recentHeaders.CurrentHeader().Hash : NativeContract.Ledger.CurrentHeaderHash(currentSnapshot); + public UInt256 CurrentHeaderHash => recentHeaders.Added ? recentHeaders.CurrentHeader().Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); private static Blockchain singleton; public static Blockchain Singleton @@ -281,13 +281,6 @@ private VerifyResult OnNewBlock(Block block) { Header header = block.Header; recentHeaders.Add(header); - using (SnapshotCache snapshot = GetSnapshot()) - { - NativeContract.Ledger.SaveHeader(snapshot, header); - NativeContract.Ledger.SetCurrentHeader(snapshot, header.Hash, header.Index); - snapshot.Commit(); - } - UpdateCurrentSnapshot(); } } return VerifyResult.Succeed; @@ -303,12 +296,8 @@ private void OnNewHeaders(Header[] headers) if (header.Index < HeaderHeight + 1) continue; if (!header.Verify(snapshot)) break; recentHeaders.Add(header); - NativeContract.Ledger.SaveHeader(snapshot, header); - NativeContract.Ledger.SetCurrentHeader(snapshot, header.Hash, header.Index); } - snapshot.Commit(); } - UpdateCurrentSnapshot(); system.TaskManager.Tell(new TaskManager.HeaderTaskCompleted(), Sender); } @@ -377,12 +366,7 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { - if (block.Index == 0) - { - NativeContract.Ledger.SetCurrentHeader(snapshot, block.Hash, block.Index); - recentHeaders.Add(block.Header); - } - else if (block.Index == HeaderHeight + 1) + if (block.Index != 0 && block.Index == HeaderHeight + 1) { recentHeaders.Add(block.Header); } @@ -474,7 +458,7 @@ private void SendRelayResult(IInventory inventory, VerifyResult result) Context.System.EventStream.Publish(rr); } - private UInt256 GetBlockHash(uint index) + public UInt256 GetBlockHash(uint index) { UInt256 hash = recentHeaders.At(index)?.Hash; return hash != null ? hash : NativeContract.Ledger.GetBlockHash(currentSnapshot, index); diff --git a/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs b/src/neo/Network/P2P/Payloads/GetBlockDataPayload.cs similarity index 100% rename from src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs rename to src/neo/Network/P2P/Payloads/GetBlockDataPayload.cs diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 380e07a62b..3f9596d482 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -267,10 +267,10 @@ private void RequestTasks(TaskSession session) { for (uint i = Blockchain.Singleton.Height + 1; i <= Blockchain.Singleton.HeaderHeight; i++) { - hash = NativeContract.Ledger.GetBlockHash(snapshot, i); + hash = Blockchain.Singleton.GetBlockHash(i); if (!globalTasks.ContainsKey(hash)) { - hash = NativeContract.Ledger.GetBlockHash(snapshot, i - 1); + hash = Blockchain.Singleton.GetBlockHash(i - 1); break; } } diff --git a/src/neo/SmartContract/Native/LedgerContract.cs b/src/neo/SmartContract/Native/LedgerContract.cs index 7a260bb141..2658584c7f 100644 --- a/src/neo/SmartContract/Native/LedgerContract.cs +++ b/src/neo/SmartContract/Native/LedgerContract.cs @@ -15,7 +15,6 @@ public sealed class LedgerContract : NativeContract private const byte Prefix_CurrentBlock = 12; private const byte Prefix_Block = 5; private const byte Prefix_Transaction = 11; - private const byte Prefix_CurrentHeader = 14; internal LedgerContract() { @@ -23,8 +22,8 @@ internal LedgerContract() internal override void OnPersist(ApplicationEngine engine) { - engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_BlockHash).AddBigEndian(engine.PersistingBlock.Index), () => new StorageItem(engine.PersistingBlock.Hash.ToArray(), true)); - engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_Block).Add(engine.PersistingBlock.Hash), () => new StorageItem(Trim(engine.PersistingBlock).ToArray(), true)); + engine.Snapshot.Add(CreateStorageKey(Prefix_BlockHash).AddBigEndian(engine.PersistingBlock.Index), new StorageItem(engine.PersistingBlock.Hash.ToArray(), true)); + engine.Snapshot.Add(CreateStorageKey(Prefix_Block).Add(engine.PersistingBlock.Hash), new StorageItem(Trim(engine.PersistingBlock).ToArray(), true)); foreach (Transaction tx in engine.PersistingBlock.Transactions) { engine.Snapshot.Add(CreateStorageKey(Prefix_Transaction).Add(tx.Hash), new StorageItem(new TransactionState @@ -40,10 +39,6 @@ internal override void PostPersist(ApplicationEngine engine) HashIndexState state = engine.Snapshot.GetAndChange(CreateStorageKey(Prefix_CurrentBlock), () => new StorageItem(new HashIndexState())).GetInteroperable(); state.Hash = engine.PersistingBlock.Hash; state.Index = engine.PersistingBlock.Index; - if (CurrentHeaderIndex(engine.Snapshot) < engine.PersistingBlock.Index) - { - SetCurrentHeader(engine.Snapshot, engine.PersistingBlock.Hash, engine.PersistingBlock.Index); - } } internal bool Initialized(DataCache snapshot) @@ -157,29 +152,6 @@ public Transaction GetTransaction(DataCache snapshot, UInt256 hash) return GetTransactionState(snapshot, hash)?.Transaction; } - public void SetCurrentHeader(DataCache snapshot, UInt256 hash, uint index) - { - HashIndexState state = snapshot.GetAndChange(CreateStorageKey(Prefix_CurrentHeader), () => new StorageItem(new HashIndexState())).GetInteroperable(); - state.Hash = hash; - state.Index = index; - } - - public UInt256 CurrentHeaderHash(DataCache snapshot) - { - return snapshot[CreateStorageKey(Prefix_CurrentHeader)].GetInteroperable().Hash; - } - - public uint CurrentHeaderIndex(DataCache snapshot) - { - return snapshot[CreateStorageKey(Prefix_CurrentHeader)].GetInteroperable().Index; - } - - public void SaveHeader(DataCache snapshot, Header header) - { - snapshot.Add(CreateStorageKey(Prefix_BlockHash).AddBigEndian(header.Index), new StorageItem(header.Hash.ToArray(), true)); - snapshot.Add(CreateStorageKey(Prefix_Block).Add(header.Hash), new StorageItem(header.Trim().ToArray(), true)); - } - [ContractMethod(0_01000000, CallFlags.ReadStates, Name = "getTransaction")] private Transaction GetTransactionForContract(DataCache snapshot, UInt256 hash) { From 8ac3ea2c07391b69a917500f14971843beae1cc8 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 19:25:59 +0800 Subject: [PATCH 16/91] fix --- src/neo/Network/P2P/Payloads/Header.cs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/neo/Network/P2P/Payloads/Header.cs b/src/neo/Network/P2P/Payloads/Header.cs index 6d76298d47..0a6ebeff3b 100644 --- a/src/neo/Network/P2P/Payloads/Header.cs +++ b/src/neo/Network/P2P/Payloads/Header.cs @@ -36,20 +36,5 @@ public override void Serialize(BinaryWriter writer) base.Serialize(writer); writer.Write((byte)0); } - - public TrimmedBlock Trim() - { - return new TrimmedBlock - { - Version = Version, - PrevHash = PrevHash, - MerkleRoot = MerkleRoot, - Timestamp = Timestamp, - Index = Index, - NextConsensus = NextConsensus, - Witness = Witness, - Hashes = Array.Empty() - }; - } } } From 538eeab26ca08a27cd41a15c12566bba47cd8e20 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 26 Jan 2021 19:27:01 +0800 Subject: [PATCH 17/91] fix --- src/neo/Network/P2P/Payloads/Header.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/neo/Network/P2P/Payloads/Header.cs b/src/neo/Network/P2P/Payloads/Header.cs index 0a6ebeff3b..301d1cf05b 100644 --- a/src/neo/Network/P2P/Payloads/Header.cs +++ b/src/neo/Network/P2P/Payloads/Header.cs @@ -1,4 +1,3 @@ -using Neo.SmartContract.Native; using System; using System.IO; From 5eacd2f53f88d630f1c3d0345f371aa2c5e9f360 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 26 Jan 2021 22:08:08 +0800 Subject: [PATCH 18/91] Update HeaderCache --- src/neo/Ledger/Blockchain.cs | 1 + src/neo/{IO/Caching => Ledger}/HeaderCache.cs | 38 +++++++++++-------- 2 files changed, 23 insertions(+), 16 deletions(-) rename src/neo/{IO/Caching => Ledger}/HeaderCache.cs (70%) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 3f2f3355a9..923801bfa1 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -439,6 +439,7 @@ private void Persist(Block block) protected override void PostStop() { base.PostStop(); + recentHeaders.Dispose(); currentSnapshot?.Dispose(); } diff --git a/src/neo/IO/Caching/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs similarity index 70% rename from src/neo/IO/Caching/HeaderCache.cs rename to src/neo/Ledger/HeaderCache.cs index 163105b66b..a86db392dc 100644 --- a/src/neo/IO/Caching/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -2,32 +2,38 @@ using System; using System.Threading; -namespace Neo.IO.Caching +namespace Neo.Ledger { - internal class HeaderCache + internal class HeaderCache : IDisposable { - private int max_capacity; + private readonly ReaderWriterLockSlim readerWriterLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + private readonly Header[] headers; + private readonly int max_capacity; private int startPos = -1; private int endPos = -1; private uint startIndex; private uint endIndex; - private Header[] headers = null; - protected readonly ReaderWriterLockSlim RwSyncRootLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + + public bool Added => startPos != -1; public HeaderCache(int max_capacity) { - if (max_capacity <= 0) throw new ArgumentException("illegal max_capacity"); + if (max_capacity <= 0) + throw new ArgumentOutOfRangeException(nameof(max_capacity)); this.max_capacity = max_capacity; - headers = new Header[max_capacity]; + this.headers = new Header[max_capacity]; } - public bool Added => startPos != -1; + public void Dispose() + { + readerWriterLock.Dispose(); + } public void Add(Header header) { + readerWriterLock.EnterWriteLock(); try { - RwSyncRootLock.EnterWriteLock(); if (!Added) { startIndex = header.Index; @@ -52,47 +58,47 @@ public void Add(Header header) } finally { - RwSyncRootLock.ExitWriteLock(); + readerWriterLock.ExitWriteLock(); } } public Header At(uint height) { + readerWriterLock.EnterReadLock(); try { - RwSyncRootLock.EnterReadLock(); if (startPos == -1 || height < startIndex || height > endIndex) return null; return headers[(startPos + height - startIndex) % max_capacity]; } finally { - RwSyncRootLock.ExitReadLock(); + readerWriterLock.ExitReadLock(); } } public uint HeaderHeight() { + readerWriterLock.EnterReadLock(); try { - RwSyncRootLock.EnterReadLock(); return endIndex; } finally { - RwSyncRootLock.ExitReadLock(); + readerWriterLock.ExitReadLock(); } } public Header CurrentHeader() { + readerWriterLock.EnterReadLock(); try { - RwSyncRootLock.EnterReadLock(); return endPos == -1 ? null : headers[endPos]; } finally { - RwSyncRootLock.ExitReadLock(); + readerWriterLock.ExitReadLock(); } } } From fa112e91ee3b7e85c41b15184d434a8505a03349 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 26 Jan 2021 22:30:45 +0800 Subject: [PATCH 19/91] Update HeaderCache.cs --- src/neo/Ledger/HeaderCache.cs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index a86db392dc..adc2215b8f 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -37,24 +37,21 @@ public void Add(Header header) if (!Added) { startIndex = header.Index; - endIndex = header.Index; startPos = 0; endPos = 0; - headers[0] = header; - return; } else { if (header.Index != headers[endPos].Index + 1) throw new ArgumentException("illegal header"); endPos = (endPos + 1) % max_capacity; - endIndex++; - } - if (endPos == startPos) - { - startPos = (startPos + 1) % max_capacity; - startIndex++; + if (endPos == startPos) + { + startPos = (startPos + 1) % max_capacity; + startIndex++; + } } headers[endPos] = header; + endIndex = header.Index; } finally { From 9808fc3cdede41a41d3857c1af7042965e54ed12 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Wed, 27 Jan 2021 00:42:17 +0800 Subject: [PATCH 20/91] update headercache logic --- src/neo/Ledger/Blockchain.cs | 2 +- src/neo/Ledger/HeaderCache.cs | 36 ++++++++++++++++++++++++++++------- 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 923801bfa1..c1c977cca1 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -368,7 +368,7 @@ private void Persist(Block block) { if (block.Index != 0 && block.Index == HeaderHeight + 1) { - recentHeaders.Add(block.Header); + recentHeaders.Remove(block.Index); } List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index adc2215b8f..a8f95dec0f 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -29,7 +29,7 @@ public void Dispose() readerWriterLock.Dispose(); } - public void Add(Header header) + public bool Add(Header header) { readerWriterLock.EnterWriteLock(); try @@ -43,15 +43,37 @@ public void Add(Header header) else { if (header.Index != headers[endPos].Index + 1) throw new ArgumentException("illegal header"); - endPos = (endPos + 1) % max_capacity; - if (endPos == startPos) - { - startPos = (startPos + 1) % max_capacity; - startIndex++; - } + var newEndPos = (endPos + 1) % max_capacity; + if (newEndPos == startPos) return false; + endPos = newEndPos; } headers[endPos] = header; endIndex = header.Index; + return true; + } + finally + { + readerWriterLock.ExitWriteLock(); + } + } + + public bool Remove(uint height) + { + readerWriterLock.EnterWriteLock(); + try + { + if (!Added) return false; + if (startIndex != height) return false; + headers[startPos] = null; + startPos = (startPos + 1) % max_capacity; + if ((endPos + 1) % max_capacity == startPos) + { + startPos = -1; + endPos = -1; + return true; + } + startIndex = headers[startPos].Index; + return true; } finally { From fdfbb675fac81cb6c453f7472ca0cab0dbb286c6 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Wed, 27 Jan 2021 14:58:13 +0800 Subject: [PATCH 21/91] fix --- src/neo/Ledger/Blockchain.cs | 19 +++++++++++++++---- src/neo/Network/P2P/Payloads/BlockBase.cs | 8 +++++--- src/neo/Network/P2P/TaskManager.cs | 4 ++-- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index c1c977cca1..c6b4a4969c 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -232,7 +232,7 @@ private VerifyResult OnNewBlock(Block block) } else { - if (!block.Hash.Equals(GetBlockHash(block.Index))) + if (!block.Hash.Equals(GetBlockHash(block.Index, currentSnapshot))) return VerifyResult.Invalid; } block_cache.TryAdd(block.Hash, block); @@ -459,10 +459,21 @@ private void SendRelayResult(IInventory inventory, VerifyResult result) Context.System.EventStream.Publish(rr); } - public UInt256 GetBlockHash(uint index) + private Header GetCachedHeader(uint index) { - UInt256 hash = recentHeaders.At(index)?.Hash; - return hash != null ? hash : NativeContract.Ledger.GetBlockHash(currentSnapshot, index); + return recentHeaders.At(index); + } + + public Header GetHeader(uint index, DataCache snapshot) + { + Header header = GetCachedHeader(index); + return header != null ? header : NativeContract.Ledger.GetHeader(snapshot, index); + } + + public UInt256 GetBlockHash(uint index, DataCache snapshot) + { + UInt256 hash = GetCachedHeader(index)?.Hash; + return hash != null ? hash : NativeContract.Ledger.GetBlockHash(snapshot, index); } private void UpdateCurrentSnapshot() diff --git a/src/neo/Network/P2P/Payloads/BlockBase.cs b/src/neo/Network/P2P/Payloads/BlockBase.cs index 08b2657295..95e98cb3ec 100644 --- a/src/neo/Network/P2P/Payloads/BlockBase.cs +++ b/src/neo/Network/P2P/Payloads/BlockBase.cs @@ -1,5 +1,6 @@ using Neo.IO; using Neo.IO.Json; +using Neo.Ledger; using Neo.Persistence; using Neo.SmartContract; using Neo.SmartContract.Native; @@ -76,8 +77,9 @@ void IVerifiable.DeserializeUnsigned(BinaryReader reader) UInt160[] IVerifiable.GetScriptHashesForVerifying(DataCache snapshot) { if (PrevHash == UInt256.Zero) return new[] { Witness.ScriptHash }; - TrimmedBlock prev = NativeContract.Ledger.GetTrimmedBlock(snapshot, PrevHash); + var prev = Blockchain.Singleton.GetHeader(Index - 1, snapshot); if (prev is null) throw new InvalidOperationException(); + if (prev.Hash != PrevHash) throw new InvalidOperationException(); return new[] { prev.NextConsensus }; } @@ -114,9 +116,9 @@ public virtual JObject ToJson() public virtual bool Verify(DataCache snapshot) { - TrimmedBlock prev = NativeContract.Ledger.GetTrimmedBlock(snapshot, PrevHash); + var prev = Blockchain.Singleton.GetHeader(Index - 1, snapshot); if (prev is null) return false; - if (prev.Index + 1 != Index) return false; + if (prev.Hash != PrevHash) return false; if (prev.Timestamp >= Timestamp) return false; if (!this.VerifyWitnesses(snapshot, 1_00000000)) return false; return true; diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 3f9596d482..5fbe30598a 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -267,10 +267,10 @@ private void RequestTasks(TaskSession session) { for (uint i = Blockchain.Singleton.Height + 1; i <= Blockchain.Singleton.HeaderHeight; i++) { - hash = Blockchain.Singleton.GetBlockHash(i); + hash = Blockchain.Singleton.GetBlockHash(i, snapshot); if (!globalTasks.ContainsKey(hash)) { - hash = Blockchain.Singleton.GetBlockHash(i - 1); + hash = Blockchain.Singleton.GetBlockHash(i - 1, snapshot); break; } } From fabf59c8a220cb2844f17ad94b6e43834e7ebc8d Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Wed, 27 Jan 2021 16:15:45 +0800 Subject: [PATCH 22/91] Add IndexedQueue --- src/neo/IO/Caching/IndexedQueue.cs | 239 +++++++++++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 src/neo/IO/Caching/IndexedQueue.cs diff --git a/src/neo/IO/Caching/IndexedQueue.cs b/src/neo/IO/Caching/IndexedQueue.cs new file mode 100644 index 0000000000..aa02dbd7f0 --- /dev/null +++ b/src/neo/IO/Caching/IndexedQueue.cs @@ -0,0 +1,239 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; + +namespace Neo.IO.Caching +{ + /// + /// Represents a queue with indexed access to the items + /// + /// The type of items in the queue + class IndexedQueue : IReadOnlyList + { + private const int DefaultCapacity = 16; + private const int GrowthFactor = 2; + private const float TrimThreshold = 0.9f; + + private T[] _array; + private int _head; + private int _count; + + /// + /// Indicates the count of items in the queue + /// + public int Count => _count; + + /// + /// Creates a queue with the default capacity + /// + public IndexedQueue() : this(DefaultCapacity) + { + } + + /// + /// Creates a queue with the specified capacity + /// + /// The initial capacity of the queue + public IndexedQueue(int capacity) + { + if (capacity <= 0) + throw new ArgumentOutOfRangeException(nameof(capacity), "The capacity must be greater than zero."); + _array = new T[capacity]; + _head = 0; + _count = 0; + } + + /// + /// Creates a queue filled with the specified items + /// + /// The collection of items to fill the queue with + public IndexedQueue(IEnumerable collection) + { + _array = collection.ToArray(); + _head = 0; + _count = _array.Length; + } + + /// + /// Gets or sets the value at the index + /// + /// The index + /// The value at the specified index + public T this[int index] + { + get + { + if (index < 0 || index >= _count) + throw new IndexOutOfRangeException(); + return _array[(index + _head) % _array.Length]; + } + set + { + if (index < 0 || index >= _count) + throw new IndexOutOfRangeException(); + _array[(index + _head) % _array.Length] = value; + } + } + + /// + /// Inserts an item at the rear of the queue + /// + /// The item to insert + public void Enqueue(T item) + { + if (_array.Length == _count) + { + int newSize = _array.Length * GrowthFactor; + if (_head == 0) + { + Array.Resize(ref _array, newSize); + } + else + { + T[] buffer = new T[newSize]; + Array.Copy(_array, _head, buffer, 0, _array.Length - _head); + Array.Copy(_array, 0, buffer, _array.Length - _head, _head); + _array = buffer; + _head = 0; + } + } + _array[(_head + _count) % _array.Length] = item; + ++_count; + } + + /// + /// Provides access to the item at the front of the queue without dequeueing it + /// + /// The frontmost item + public T Peek() + { + if (_count == 0) + throw new InvalidOperationException("The queue is empty."); + return _array[_head]; + } + + /// + /// Attempts to return an item from the front of the queue without removing it + /// + /// The item + /// True if the queue returned an item or false if the queue is empty + public bool TryPeek(out T item) + { + if (_count == 0) + { + item = default; + return false; + } + else + { + item = _array[_head]; + return true; + } + } + + /// + /// Removes an item from the front of the queue, returning it + /// + /// The item that was removed + public T Dequeue() + { + if (_count == 0) + throw new InvalidOperationException("The queue is empty"); + T result = _array[_head]; + ++_head; + _head %= _array.Length; + --_count; + return result; + } + + /// + /// Attempts to return an item from the front of the queue, removing it + /// + /// The item + /// True if the queue returned an item or false if the queue is empty + public bool TryDequeue(out T item) + { + if (_count == 0) + { + item = default; + return false; + } + else + { + item = _array[_head]; + ++_head; + _head %= _array.Length; + --_count; + return true; + } + } + + /// + /// Clears the items from the queue + /// + public void Clear() + { + _head = 0; + _count = 0; + } + + /// + /// Trims the extra array space that isn't being used. + /// + public void TrimExcess() + { + if (_count == 0) + { + _array = new T[DefaultCapacity]; + } + else if (_array.Length * TrimThreshold >= _count) + { + T[] arr = new T[_count]; + CopyTo(arr, 0); + _array = arr; + _head = 0; + } + } + + /// + /// Copys the queue's items to a destination array + /// + /// The destination array + /// The index in the destination to start copying at + public void CopyTo(T[] array, int arrayIndex) + { + if (array is null) throw new ArgumentNullException(nameof(array)); + if (arrayIndex < 0 || arrayIndex + _count > array.Length) + throw new ArgumentOutOfRangeException(nameof(arrayIndex)); + if (_head + _count <= _array.Length) + { + Array.Copy(_array, _head, array, arrayIndex, _count); + } + else + { + Array.Copy(_array, _head, array, arrayIndex, _array.Length - _head); + Array.Copy(_array, 0, array, arrayIndex + _array.Length - _head, _count + _head - _array.Length); + } + } + + /// + /// Returns an array of the items in the queue + /// + /// An array containing the queue's items + public T[] ToArray() + { + T[] result = new T[_count]; + CopyTo(result, 0); + return result; + } + + public IEnumerator GetEnumerator() + { + for (int i = 0; i < _count; i++) + yield return _array[(_head + i) % _array.Length]; + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } +} From 4143d8203f6d836e1d99ada389740421e9ae52fb Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Wed, 27 Jan 2021 16:42:56 +0800 Subject: [PATCH 23/91] Remove HeaderCache --- src/neo/Ledger/Blockchain.cs | 28 ++++---- src/neo/Ledger/HeaderCache.cs | 124 ---------------------------------- 2 files changed, 13 insertions(+), 139 deletions(-) delete mode 100644 src/neo/Ledger/HeaderCache.cs diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index c6b4a4969c..97310eadfb 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -65,7 +65,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private readonly ConcurrentDictionary block_cache = new ConcurrentDictionary(); private readonly Dictionary block_cache_unverified = new Dictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); - private readonly HeaderCache recentHeaders = new HeaderCache(10000); + private readonly IndexedQueue
recentHeaders = new IndexedQueue
(); private SnapshotCache currentSnapshot; private ImmutableHashSet extensibleWitnessWhiteList; @@ -74,9 +74,9 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); - public uint HeaderHeight => recentHeaders.Added ? recentHeaders.HeaderHeight() : NativeContract.Ledger.CurrentIndex(currentSnapshot); + public uint HeaderHeight => recentHeaders.Count > 0 ? recentHeaders[^1].Index : NativeContract.Ledger.CurrentIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); - public UInt256 CurrentHeaderHash => recentHeaders.Added ? recentHeaders.CurrentHeader().Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); + public UInt256 CurrentHeaderHash => recentHeaders.Count > 0 ? recentHeaders[^1].Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); private static Blockchain singleton; public static Blockchain Singleton @@ -244,7 +244,7 @@ private VerifyResult OnNewBlock(Block block) { blocksToPersistList.Add(block_persist); if (block_persist.Index + 1 > HeaderHeight) break; - UInt256 hash = recentHeaders.At(block_persist.Index + 1).Hash; + UInt256 hash = GetCachedHeader(block_persist.Index + 1).Hash; if (!block_cache.TryGetValue(hash, out block_persist)) break; } @@ -278,10 +278,7 @@ private VerifyResult OnNewBlock(Block block) if (block.Index + 99 >= HeaderHeight) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = block }); if (block.Index == HeaderHeight + 1) - { - Header header = block.Header; - recentHeaders.Add(header); - } + recentHeaders.Enqueue(block.Header); } return VerifyResult.Succeed; } @@ -295,7 +292,7 @@ private void OnNewHeaders(Header[] headers) if (header.Index > HeaderHeight + 1) break; if (header.Index < HeaderHeight + 1) continue; if (!header.Verify(snapshot)) break; - recentHeaders.Add(header); + recentHeaders.Enqueue(header); } } system.TaskManager.Tell(new TaskManager.HeaderTaskCompleted(), Sender); @@ -366,10 +363,7 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { - if (block.Index != 0 && block.Index == HeaderHeight + 1) - { - recentHeaders.Remove(block.Index); - } + recentHeaders.TryDequeue(out _); List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) { @@ -439,7 +433,6 @@ private void Persist(Block block) protected override void PostStop() { base.PostStop(); - recentHeaders.Dispose(); currentSnapshot?.Dispose(); } @@ -461,7 +454,12 @@ private void SendRelayResult(IInventory inventory, VerifyResult result) private Header GetCachedHeader(uint index) { - return recentHeaders.At(index); + if (recentHeaders.Count == 0) return null; + uint firstIndex = recentHeaders[0].Index; + if (index < firstIndex) return null; + index -= firstIndex; + if (index >= recentHeaders.Count) return null; + return recentHeaders[(int)index]; } public Header GetHeader(uint index, DataCache snapshot) diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs deleted file mode 100644 index a8f95dec0f..0000000000 --- a/src/neo/Ledger/HeaderCache.cs +++ /dev/null @@ -1,124 +0,0 @@ -using Neo.Network.P2P.Payloads; -using System; -using System.Threading; - -namespace Neo.Ledger -{ - internal class HeaderCache : IDisposable - { - private readonly ReaderWriterLockSlim readerWriterLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); - private readonly Header[] headers; - private readonly int max_capacity; - private int startPos = -1; - private int endPos = -1; - private uint startIndex; - private uint endIndex; - - public bool Added => startPos != -1; - - public HeaderCache(int max_capacity) - { - if (max_capacity <= 0) - throw new ArgumentOutOfRangeException(nameof(max_capacity)); - this.max_capacity = max_capacity; - this.headers = new Header[max_capacity]; - } - - public void Dispose() - { - readerWriterLock.Dispose(); - } - - public bool Add(Header header) - { - readerWriterLock.EnterWriteLock(); - try - { - if (!Added) - { - startIndex = header.Index; - startPos = 0; - endPos = 0; - } - else - { - if (header.Index != headers[endPos].Index + 1) throw new ArgumentException("illegal header"); - var newEndPos = (endPos + 1) % max_capacity; - if (newEndPos == startPos) return false; - endPos = newEndPos; - } - headers[endPos] = header; - endIndex = header.Index; - return true; - } - finally - { - readerWriterLock.ExitWriteLock(); - } - } - - public bool Remove(uint height) - { - readerWriterLock.EnterWriteLock(); - try - { - if (!Added) return false; - if (startIndex != height) return false; - headers[startPos] = null; - startPos = (startPos + 1) % max_capacity; - if ((endPos + 1) % max_capacity == startPos) - { - startPos = -1; - endPos = -1; - return true; - } - startIndex = headers[startPos].Index; - return true; - } - finally - { - readerWriterLock.ExitWriteLock(); - } - } - - public Header At(uint height) - { - readerWriterLock.EnterReadLock(); - try - { - if (startPos == -1 || height < startIndex || height > endIndex) return null; - return headers[(startPos + height - startIndex) % max_capacity]; - } - finally - { - readerWriterLock.ExitReadLock(); - } - } - - public uint HeaderHeight() - { - readerWriterLock.EnterReadLock(); - try - { - return endIndex; - } - finally - { - readerWriterLock.ExitReadLock(); - } - } - - public Header CurrentHeader() - { - readerWriterLock.EnterReadLock(); - try - { - return endPos == -1 ? null : headers[endPos]; - } - finally - { - readerWriterLock.ExitReadLock(); - } - } - } -} From e7529cfcb6c29dbffc9458e19a7f2ae3093e675b Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Wed, 27 Jan 2021 16:50:05 +0800 Subject: [PATCH 24/91] Rename --- src/neo/Ledger/Blockchain.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 97310eadfb..28917fd98e 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -62,10 +62,10 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private static readonly object lockObj = new object(); private readonly NeoSystem system; private readonly IActorRef txrouter; + private readonly IndexedQueue
header_cache = new IndexedQueue
(); private readonly ConcurrentDictionary block_cache = new ConcurrentDictionary(); private readonly Dictionary block_cache_unverified = new Dictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); - private readonly IndexedQueue
recentHeaders = new IndexedQueue
(); private SnapshotCache currentSnapshot; private ImmutableHashSet extensibleWitnessWhiteList; @@ -74,9 +74,9 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); - public uint HeaderHeight => recentHeaders.Count > 0 ? recentHeaders[^1].Index : NativeContract.Ledger.CurrentIndex(currentSnapshot); + public uint HeaderHeight => header_cache.Count > 0 ? header_cache[^1].Index : NativeContract.Ledger.CurrentIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); - public UInt256 CurrentHeaderHash => recentHeaders.Count > 0 ? recentHeaders[^1].Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); + public UInt256 CurrentHeaderHash => header_cache.Count > 0 ? header_cache[^1].Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); private static Blockchain singleton; public static Blockchain Singleton @@ -278,7 +278,7 @@ private VerifyResult OnNewBlock(Block block) if (block.Index + 99 >= HeaderHeight) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = block }); if (block.Index == HeaderHeight + 1) - recentHeaders.Enqueue(block.Header); + header_cache.Enqueue(block.Header); } return VerifyResult.Succeed; } @@ -292,7 +292,7 @@ private void OnNewHeaders(Header[] headers) if (header.Index > HeaderHeight + 1) break; if (header.Index < HeaderHeight + 1) continue; if (!header.Verify(snapshot)) break; - recentHeaders.Enqueue(header); + header_cache.Enqueue(header); } } system.TaskManager.Tell(new TaskManager.HeaderTaskCompleted(), Sender); @@ -363,7 +363,7 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { - recentHeaders.TryDequeue(out _); + header_cache.TryDequeue(out _); List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) { @@ -454,12 +454,12 @@ private void SendRelayResult(IInventory inventory, VerifyResult result) private Header GetCachedHeader(uint index) { - if (recentHeaders.Count == 0) return null; - uint firstIndex = recentHeaders[0].Index; + if (header_cache.Count == 0) return null; + uint firstIndex = header_cache[0].Index; if (index < firstIndex) return null; index -= firstIndex; - if (index >= recentHeaders.Count) return null; - return recentHeaders[(int)index]; + if (index >= header_cache.Count) return null; + return header_cache[(int)index]; } public Header GetHeader(uint index, DataCache snapshot) From b7075b5981cf237b425eaec220a10e71f41b0bb1 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Wed, 27 Jan 2021 16:55:09 +0800 Subject: [PATCH 25/91] Limit the count of cached headers --- src/neo/Ledger/Blockchain.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 28917fd98e..58cebe84bc 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -285,6 +285,7 @@ private VerifyResult OnNewBlock(Block block) private void OnNewHeaders(Header[] headers) { + if (header_cache.Count >= 10000) return; using (SnapshotCache snapshot = GetSnapshot()) { foreach (Header header in headers) From 2ca2e5959095e0e4e9cf63fe11723c52a517eb87 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 27 Jan 2021 09:56:23 +0100 Subject: [PATCH 26/91] Clean using --- src/neo/Network/P2P/Payloads/BlockBase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/Payloads/BlockBase.cs b/src/neo/Network/P2P/Payloads/BlockBase.cs index 95e98cb3ec..14233238b5 100644 --- a/src/neo/Network/P2P/Payloads/BlockBase.cs +++ b/src/neo/Network/P2P/Payloads/BlockBase.cs @@ -3,7 +3,6 @@ using Neo.Ledger; using Neo.Persistence; using Neo.SmartContract; -using Neo.SmartContract.Native; using Neo.Wallets; using System; using System.IO; @@ -118,6 +117,7 @@ public virtual bool Verify(DataCache snapshot) { var prev = Blockchain.Singleton.GetHeader(Index - 1, snapshot); if (prev is null) return false; + if (prev.Index + 1 != Index) return false; if (prev.Hash != PrevHash) return false; if (prev.Timestamp >= Timestamp) return false; if (!this.VerifyWitnesses(snapshot, 1_00000000)) return false; From b2574f6d62398986d0724227c6809561fb70fd57 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 27 Jan 2021 10:37:21 +0100 Subject: [PATCH 27/91] Add IndexedQueue UT --- .../IO/Caching/UT_IndexedQueue.cs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs diff --git a/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs b/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs new file mode 100644 index 0000000000..e09726924c --- /dev/null +++ b/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs @@ -0,0 +1,98 @@ +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO.Caching; +using System; +using System.Linq; + +namespace Neo.UnitTests.IO.Caching +{ + [TestClass] + public class UT_IndexedQueue + { + [TestMethod] + public void TestDefault() + { + var queue = new IndexedQueue(10); + queue.Count.Should().Be(0); + + queue = new IndexedQueue(); + queue.Count.Should().Be(0); + queue.TrimExcess(); + queue.Count.Should().Be(0); + + queue = new IndexedQueue(Array.Empty()); + queue.Count.Should().Be(0); + queue.TryPeek(out var a).Should().BeFalse(); + a.Should().Be(0); + queue.TryDequeue(out a).Should().BeFalse(); + a.Should().Be(0); + + Assert.ThrowsException(() => queue.Peek()); + Assert.ThrowsException(() => queue.Dequeue()); + Assert.ThrowsException(() => _ = queue[-1]); + Assert.ThrowsException(() => queue[-1] = 1); + Assert.ThrowsException(() => _ = queue[1]); + Assert.ThrowsException(() => queue[1] = 1); + Assert.ThrowsException(() => new IndexedQueue(-1)); + } + + [TestMethod] + public void TestQueue() + { + var queue = new IndexedQueue(new int[] { 1, 2, 3 }); + queue.Count.Should().Be(3); + + queue.Enqueue(4); + queue.Count.Should().Be(4); + queue.Peek().Should().Be(1); + queue.TryPeek(out var a).Should().BeTrue(); + a.Should().Be(1); + + queue[0].Should().Be(1); + queue[1].Should().Be(2); + queue[2].Should().Be(3); + queue.Dequeue().Should().Be(1); + queue.Dequeue().Should().Be(2); + queue.Dequeue().Should().Be(3); + queue[0] = 5; + queue.TryDequeue(out a).Should().BeTrue(); + a.Should().Be(5); + + queue.Enqueue(4); + queue.Clear(); + queue.Count.Should().Be(0); + } + + [TestMethod] + public void TestEnumerator() + { + int[] arr = new int[3] { 1, 2, 3 }; + var queue = new IndexedQueue(arr); + + arr.SequenceEqual(queue).Should().BeTrue(); + } + + [TestMethod] + public void TestCopyTo() + { + int[] arr = new int[3]; + var queue = new IndexedQueue(new int[] { 1, 2, 3 }); + + Assert.ThrowsException(() => queue.CopyTo(null, 0)); + Assert.ThrowsException(() => queue.CopyTo(arr, -1)); + Assert.ThrowsException(() => queue.CopyTo(arr, 2)); + + queue.CopyTo(arr, 0); + + arr[0].Should().Be(1); + arr[1].Should().Be(2); + arr[2].Should().Be(3); + + arr = queue.ToArray(); + + arr[0].Should().Be(1); + arr[1].Should().Be(2); + arr[2].Should().Be(3); + } + } +} From fe294641354f43e7db269f00d0dd180f9d58273a Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Wed, 27 Jan 2021 19:18:00 +0800 Subject: [PATCH 28/91] restore GetBlockByIndexPayload and fix --- src/neo/Ledger/Blockchain.cs | 1 + src/neo/Network/P2P/MessageCommand.cs | 4 +- ...taPayload.cs => GetBlockByIndexPayload.cs} | 16 ++++---- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 9 ++-- src/neo/Network/P2P/TaskManager.cs | 2 +- .../P2P/Payloads/UT_GetBlockByIndexPayload.cs | 41 +++++++++++++++++++ .../P2P/Payloads/UT_GetBlockDataPayload.cs | 38 ----------------- 7 files changed, 58 insertions(+), 53 deletions(-) rename src/neo/Network/P2P/Payloads/{GetBlockDataPayload.cs => GetBlockByIndexPayload.cs} (54%) create mode 100644 tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs delete mode 100644 tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 58cebe84bc..1d45773f55 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -77,6 +77,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public uint HeaderHeight => header_cache.Count > 0 ? header_cache[^1].Index : NativeContract.Ledger.CurrentIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); public UInt256 CurrentHeaderHash => header_cache.Count > 0 ? header_cache[^1].Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); + public int HeaderCacheCount => header_cache.Count; private static Blockchain singleton; public static Blockchain Singleton diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index d04bdb7a3e..e81504d5f9 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -31,8 +31,8 @@ public enum MessageCommand : byte Inv = 0x27, [ReflectionCache(typeof(InvPayload))] GetData = 0x28, - [ReflectionCache(typeof(GetBlockDataPayload))] - GetBlockData = 0x29, + [ReflectionCache(typeof(GetBlockByIndexPayload))] + GetBlockByIndex = 0x29, [ReflectionCache(typeof(InvPayload))] NotFound = 0x2a, [ReflectionCache(typeof(Transaction))] diff --git a/src/neo/Network/P2P/Payloads/GetBlockDataPayload.cs b/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs similarity index 54% rename from src/neo/Network/P2P/Payloads/GetBlockDataPayload.cs rename to src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs index 9ccd534f34..c07a8a8c0a 100644 --- a/src/neo/Network/P2P/Payloads/GetBlockDataPayload.cs +++ b/src/neo/Network/P2P/Payloads/GetBlockByIndexPayload.cs @@ -4,17 +4,16 @@ namespace Neo.Network.P2P.Payloads { - public class GetBlockDataPayload : ISerializable + public class GetBlockByIndexPayload : ISerializable { - private const ushort MaxBlocksCount = 500; public uint IndexStart; - public ushort Count; + public short Count; - public int Size => sizeof(uint) + sizeof(ushort); + public int Size => sizeof(uint) + sizeof(short); - public static GetBlockDataPayload Create(uint index_start, ushort count) + public static GetBlockByIndexPayload Create(uint index_start, short count = -1) { - return new GetBlockDataPayload + return new GetBlockByIndexPayload { IndexStart = index_start, Count = count @@ -24,8 +23,9 @@ public static GetBlockDataPayload Create(uint index_start, ushort count) void ISerializable.Deserialize(BinaryReader reader) { IndexStart = reader.ReadUInt32(); - Count = reader.ReadUInt16(); - if (Count == 0 || Count > MaxBlocksCount) throw new FormatException(); + Count = reader.ReadInt16(); + if (Count < -1 || Count == 0 || Count > HeadersPayload.MaxHeadersCount) + throw new FormatException(); } void ISerializable.Serialize(BinaryWriter writer) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index c80c036b19..f550d74b1b 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -81,8 +81,8 @@ private void OnMessage(Message msg) case MessageCommand.GetBlocks: OnGetBlocksMessageReceived((GetBlocksPayload)msg.Payload); break; - case MessageCommand.GetBlockData: - OnGetBlockDataMessageReceived((GetBlockDataPayload)msg.Payload); + case MessageCommand.GetBlockByIndex: + OnGetBlockByIndexMessageReceived((GetBlockByIndexPayload)msg.Payload); break; case MessageCommand.GetData: OnGetDataMessageReceived((InvPayload)msg.Payload); @@ -194,9 +194,10 @@ private void OnGetBlocksMessageReceived(GetBlocksPayload payload) EnqueueMessage(Message.Create(MessageCommand.Inv, InvPayload.Create(InventoryType.Block, hashes.ToArray()))); } - private void OnGetBlockDataMessageReceived(GetBlockDataPayload payload) + private void OnGetBlockByIndexMessageReceived(GetBlockByIndexPayload payload) { - for (uint i = payload.IndexStart, max = payload.IndexStart + payload.Count; i < max; i++) + uint count = payload.Count == -1 ? InvPayload.MaxHashesCount : Math.Min((uint)payload.Count, InvPayload.MaxHashesCount); + for (uint i = payload.IndexStart, max = payload.IndexStart + count; i < max; i++) { Block block = NativeContract.Ledger.GetBlock(Blockchain.Singleton.View, i); if (block == null) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 5fbe30598a..e5e93474f8 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -254,7 +254,7 @@ private void RequestTasks(TaskSession session) // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks - if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && Blockchain.Singleton.HeaderHeight < session.LastBlockIndex) + if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && Blockchain.Singleton.HeaderHeight < session.LastBlockIndex && Blockchain.Singleton.HeaderCacheCount < 10000) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); diff --git a/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs b/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs new file mode 100644 index 0000000000..76ed1df9f9 --- /dev/null +++ b/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockByIndexPayload.cs @@ -0,0 +1,41 @@ +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.IO; +using Neo.Network.P2P.Payloads; +using System; + +namespace Neo.UnitTests.Network.P2P.Payloads +{ + [TestClass] + public class UT_GetBlockByIndexPayload + { + [TestMethod] + public void Size_Get() + { + var test = new GetBlockByIndexPayload() { Count = 5, IndexStart = 5 }; + test.Size.Should().Be(6); + + test = GetBlockByIndexPayload.Create(1, short.MaxValue); + test.Size.Should().Be(6); + } + + [TestMethod] + public void DeserializeAndSerialize() + { + var test = new GetBlockByIndexPayload() { Count = -1, IndexStart = int.MaxValue }; + var clone = test.ToArray().AsSerializable(); + + Assert.AreEqual(test.Count, clone.Count); + Assert.AreEqual(test.IndexStart, clone.IndexStart); + + test = new GetBlockByIndexPayload() { Count = -2, IndexStart = int.MaxValue }; + Assert.ThrowsException(() => test.ToArray().AsSerializable()); + + test = new GetBlockByIndexPayload() { Count = 0, IndexStart = int.MaxValue }; + Assert.ThrowsException(() => test.ToArray().AsSerializable()); + + test = new GetBlockByIndexPayload() { Count = HeadersPayload.MaxHeadersCount + 1, IndexStart = int.MaxValue }; + Assert.ThrowsException(() => test.ToArray().AsSerializable()); + } + } +} diff --git a/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs b/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs deleted file mode 100644 index b6e6df2839..0000000000 --- a/tests/neo.UnitTests/Network/P2P/Payloads/UT_GetBlockDataPayload.cs +++ /dev/null @@ -1,38 +0,0 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.IO; -using Neo.Network.P2P.Payloads; -using System; - -namespace Neo.UnitTests.Network.P2P.Payloads -{ - [TestClass] - public class UT_GetBlockDataPayload - { - [TestMethod] - public void Size_Get() - { - var test = new GetBlockDataPayload() { Count = 5, IndexStart = 5 }; - test.Size.Should().Be(6); - - test = GetBlockDataPayload.Create(1, ushort.MaxValue); - test.Size.Should().Be(6); - } - - [TestMethod] - public void DeserializeAndSerialize() - { - var test = new GetBlockDataPayload() { Count = 1, IndexStart = int.MaxValue }; - var clone = test.ToArray().AsSerializable(); - - Assert.AreEqual(test.Count, clone.Count); - Assert.AreEqual(test.IndexStart, clone.IndexStart); - - test = new GetBlockDataPayload() { Count = 0, IndexStart = int.MaxValue }; - Assert.ThrowsException(() => test.ToArray().AsSerializable()); - - test = new GetBlockDataPayload() { Count = 501, IndexStart = int.MaxValue }; - Assert.ThrowsException(() => test.ToArray().AsSerializable()); - } - } -} From 6d8dcaeefa0644156449ecfc0c4e774eb8701fee Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Thu, 28 Jan 2021 10:53:49 +0800 Subject: [PATCH 29/91] restore getheaders --- src/neo/Network/P2P/MessageCommand.cs | 2 +- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 25 +++++++++---------- src/neo/Network/P2P/TaskManager.cs | 2 +- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index e81504d5f9..c66c15799a 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -20,7 +20,7 @@ public enum MessageCommand : byte Pong = 0x19, //synchronization - [ReflectionCache(typeof(GetBlocksPayload))] + [ReflectionCache(typeof(GetBlockByIndexPayload))] GetHeaders = 0x20, [ReflectionCache(typeof(HeadersPayload))] Headers = 0x21, diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index f550d74b1b..197bbe0248 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -88,7 +88,7 @@ private void OnMessage(Message msg) OnGetDataMessageReceived((InvPayload)msg.Payload); break; case MessageCommand.GetHeaders: - OnGetHeadersMessageReceived((GetBlocksPayload)msg.Payload); + OnGetHeadersMessageReceived((GetBlockByIndexPayload)msg.Payload); break; case MessageCommand.Headers: OnHeadersMessageReceived((HeadersPayload)msg.Payload); @@ -269,28 +269,27 @@ private void OnGetDataMessageReceived(InvPayload payload) /// /// Will be triggered when a MessageCommand.GetHeaders message is received. - /// Tell the specified number of blocks' headers starting with the requested HashStart to RemoteNode actor. + /// Tell the specified number of blocks' headers starting with the requested IndexStart to RemoteNode actor. /// A limit set by HeadersPayload.MaxHeadersCount is also applied to the number of requested Headers, namely payload.Count. /// - /// A GetBlocksPayload including start block Hash and number of blocks' headers requested. - private void OnGetHeadersMessageReceived(GetBlocksPayload payload) + /// A GetBlockByIndexPayload including start block index and number of blocks' headers requested. + private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) { - UInt256 hash = payload.HashStart; - int count = payload.Count < 0 || payload.Count > HeadersPayload.MaxHeadersCount ? HeadersPayload.MaxHeadersCount : payload.Count; + uint index = payload.IndexStart; + if (index > Blockchain.Singleton.Height) return; + List
headers = new List
(); using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) { - Block block = NativeContract.Ledger.GetBlock(snapshot, hash); - if (block == null) return; - List
headers = new List
(); - for (uint i = 1; i <= count; i++) + uint count = payload.Count == -1 || payload.Count > HeadersPayload.MaxHeadersCount ? HeadersPayload.MaxHeadersCount : (uint)payload.Count; + for (uint i = 0; i < count; i++) { - var header = NativeContract.Ledger.GetHeader(snapshot, block.Index + i); + var header = NativeContract.Ledger.GetHeader(snapshot, index + i); if (header == null) break; headers.Add(header); } - if (headers.Count == 0) return; - EnqueueMessage(Message.Create(MessageCommand.Headers, HeadersPayload.Create(headers.ToArray()))); } + if (headers.Count == 0) return; + EnqueueMessage(Message.Create(MessageCommand.Headers, HeadersPayload.Create(headers.ToArray()))); } private void OnHeadersMessageReceived(HeadersPayload payload) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index e5e93474f8..e389d8f794 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -258,7 +258,7 @@ private void RequestTasks(TaskSession session) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlocksPayload.Create(Blockchain.Singleton.CurrentHeaderHash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlockByIndexPayload.Create(Blockchain.Singleton.HeaderHeight))); } else if (Blockchain.Singleton.Height < session.LastBlockIndex) { From ccb0b45d85213c232f71fc162b7ee7b51fc9ee59 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Thu, 28 Jan 2021 15:05:32 +0800 Subject: [PATCH 30/91] Optimize IndexedQueue --- src/neo/IO/Caching/IndexedQueue.cs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/neo/IO/Caching/IndexedQueue.cs b/src/neo/IO/Caching/IndexedQueue.cs index aa02dbd7f0..b01c6ffea9 100644 --- a/src/neo/IO/Caching/IndexedQueue.cs +++ b/src/neo/IO/Caching/IndexedQueue.cs @@ -9,7 +9,7 @@ namespace Neo.IO.Caching /// Represents a queue with indexed access to the items /// /// The type of items in the queue - class IndexedQueue : IReadOnlyList + class IndexedQueue : IReadOnlyCollection { private const int DefaultCapacity = 16; private const int GrowthFactor = 2; @@ -56,23 +56,17 @@ public IndexedQueue(IEnumerable collection) } /// - /// Gets or sets the value at the index + /// Gets the value at the index /// /// The index /// The value at the specified index - public T this[int index] + public ref T this[int index] { get { if (index < 0 || index >= _count) throw new IndexOutOfRangeException(); - return _array[(index + _head) % _array.Length]; - } - set - { - if (index < 0 || index >= _count) - throw new IndexOutOfRangeException(); - _array[(index + _head) % _array.Length] = value; + return ref _array[(index + _head) % _array.Length]; } } From b563d1da75361282149b7be88b5690afb4be94cc Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Thu, 28 Jan 2021 15:20:56 +0800 Subject: [PATCH 31/91] HeaderCacheFull --- src/neo/Ledger/Blockchain.cs | 5 ++--- src/neo/Network/P2P/TaskManager.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 1d45773f55..9a39b1205e 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -66,7 +66,6 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private readonly ConcurrentDictionary block_cache = new ConcurrentDictionary(); private readonly Dictionary block_cache_unverified = new Dictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); - private SnapshotCache currentSnapshot; private ImmutableHashSet extensibleWitnessWhiteList; @@ -77,7 +76,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu public uint HeaderHeight => header_cache.Count > 0 ? header_cache[^1].Index : NativeContract.Ledger.CurrentIndex(currentSnapshot); public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); public UInt256 CurrentHeaderHash => header_cache.Count > 0 ? header_cache[^1].Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); - public int HeaderCacheCount => header_cache.Count; + public bool HeaderCacheFull => header_cache.Count >= 10000; private static Blockchain singleton; public static Blockchain Singleton @@ -286,7 +285,7 @@ private VerifyResult OnNewBlock(Block block) private void OnNewHeaders(Header[] headers) { - if (header_cache.Count >= 10000) return; + if (HeaderCacheFull) return; using (SnapshotCache snapshot = GetSnapshot()) { foreach (Header header in headers) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index e389d8f794..6b9c311707 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -254,7 +254,7 @@ private void RequestTasks(TaskSession session) // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks - if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && Blockchain.Singleton.HeaderHeight < session.LastBlockIndex && Blockchain.Singleton.HeaderCacheCount < 10000) + if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && Blockchain.Singleton.HeaderHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCacheFull) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); From 2de91fe8863381ce4f5ec2d45529f4ccbfb1a3c8 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Thu, 28 Jan 2021 15:21:39 +0800 Subject: [PATCH 32/91] Update src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 197bbe0248..359cb369fb 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -280,7 +280,7 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) List
headers = new List
(); using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) { - uint count = payload.Count == -1 || payload.Count > HeadersPayload.MaxHeadersCount ? HeadersPayload.MaxHeadersCount : (uint)payload.Count; + uint count = payload.Count == -1 ? HeadersPayload.MaxHeadersCount : (uint)payload.Count; for (uint i = 0; i < count; i++) { var header = NativeContract.Ledger.GetHeader(snapshot, index + i); From 478c12c7e6f1fbc8f696c64336f4cc7c3f7b1a11 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Thu, 28 Jan 2021 22:29:48 +0800 Subject: [PATCH 33/91] fix --- src/neo/Ledger/Blockchain.cs | 28 +++++++------- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 4 +- src/neo/Network/P2P/TaskManager.cs | 37 ++++++++----------- 3 files changed, 31 insertions(+), 38 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 9ac5174414..0d5c7367eb 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -75,10 +75,8 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu /// public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } - public uint Height => NativeContract.Ledger.CurrentIndex(currentSnapshot); - public uint HeaderHeight => header_cache.Count > 0 ? header_cache[^1].Index : NativeContract.Ledger.CurrentIndex(currentSnapshot); - public UInt256 CurrentBlockHash => NativeContract.Ledger.CurrentHash(currentSnapshot); - public UInt256 CurrentHeaderHash => header_cache.Count > 0 ? header_cache[^1].Hash : NativeContract.Ledger.CurrentHash(currentSnapshot); + public uint HeaderHeight(DataCache snapshot) => header_cache.Count > 0 ? header_cache[^1].Index : NativeContract.Ledger.CurrentIndex(snapshot); + public UInt256 CurrentHeaderHash(DataCache snapshot) => header_cache.Count > 0 ? header_cache[^1].Hash : NativeContract.Ledger.CurrentHash(snapshot); public bool HeaderCacheFull => header_cache.Count >= 10000; private static Blockchain singleton; @@ -229,30 +227,30 @@ private VerifyResult OnNewBlock(Block block) uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); if (block.Index <= currentHeight) return VerifyResult.AlreadyExists; - if (block.Index - 1 > HeaderHeight) + if (block.Index - 1 > HeaderHeight(snapshot)) { AddUnverifiedBlockToCache(block); return VerifyResult.UnableToVerify; } - if (block.Index == HeaderHeight + 1) + if (block.Index == HeaderHeight(snapshot) + 1) { if (!block.Verify(snapshot)) return VerifyResult.Invalid; } else { - if (!block.Hash.Equals(GetBlockHash(block.Index, currentSnapshot))) + if (!block.Hash.Equals(GetBlockHash(block.Index, snapshot))) return VerifyResult.Invalid; } block_cache.TryAdd(block.Hash, block); - if (block.Index == Height + 1) + if (block.Index == currentHeight + 1) { Block block_persist = block; List blocksToPersistList = new List(); while (true) { blocksToPersistList.Add(block_persist); - if (block_persist.Index + 1 > HeaderHeight) break; + if (block_persist.Index + 1 > HeaderHeight(snapshot)) break; UInt256 hash = GetCachedHeader(block_persist.Index + 1).Hash; if (!block_cache.TryGetValue(hash, out block_persist)) break; } @@ -270,10 +268,10 @@ private VerifyResult OnNewBlock(Block block) // Empirically calibrated for relaying the most recent 2 blocks persisted with 15s network // Increase in the rate of 1 block per second in configurations with faster blocks - if (blockToPersist.Index + 99 >= HeaderHeight) + if (blockToPersist.Index + 99 >= HeaderHeight(snapshot)) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = blockToPersist }); } - if (block_cache_unverified.TryGetValue(Height + 1, out var unverifiedBlocks)) + if (block_cache_unverified.TryGetValue(currentHeight + 1, out var unverifiedBlocks)) { foreach (var unverifiedBlock in unverifiedBlocks.Blocks) Self.Tell(unverifiedBlock, ActorRefs.NoSender); @@ -284,9 +282,9 @@ private VerifyResult OnNewBlock(Block block) } else { - if (block.Index + 99 >= HeaderHeight) + if (block.Index + 99 >= HeaderHeight(snapshot)) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = block }); - if (block.Index == HeaderHeight + 1) + if (block.Index == HeaderHeight(snapshot) + 1) header_cache.Enqueue(block.Header); } return VerifyResult.Succeed; @@ -299,8 +297,8 @@ private void OnNewHeaders(Header[] headers) { foreach (Header header in headers) { - if (header.Index > HeaderHeight + 1) break; - if (header.Index < HeaderHeight + 1) continue; + if (header.Index > HeaderHeight(snapshot) + 1) break; + if (header.Index < HeaderHeight(snapshot) + 1) continue; if (!header.Verify(snapshot)) break; header_cache.Enqueue(header); } diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 9101cf0f67..085706945d 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -298,7 +298,7 @@ private void OnInventoryReceived(IInventory inventory) pendingKnownHashes.Remove(inventory.Hash); if (inventory is Block block) { - if (block.Index > Blockchain.Singleton.Height + InvPayload.MaxHashesCount) return; + if (block.Index > NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View) + InvPayload.MaxHashesCount) return; UpdateLastBlockIndex(block.Index); } knownHashes.Add(inventory.Hash); @@ -340,7 +340,7 @@ private void OnMemPoolMessageReceived() private void OnPingMessageReceived(PingPayload payload) { UpdateLastBlockIndex(payload.LastBlockIndex); - EnqueueMessage(Message.Create(MessageCommand.Pong, PingPayload.Create(Blockchain.Singleton.Height, payload.Nonce))); + EnqueueMessage(Message.Create(MessageCommand.Pong, PingPayload.Create(NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View), payload.Nonce))); } private void OnPongMessageReceived(PingPayload payload) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 7e50204963..2812da956e 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -46,7 +46,6 @@ public TaskManager(NeoSystem system) { this.system = system; this.knownHashes = new HashSetCache(Blockchain.Singleton.MemPool.Capacity * 2 / 5); - this.lastTaskIndex = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); Context.System.EventStream.Subscribe(Self, typeof(Blockchain.PersistCompleted)); Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } @@ -65,7 +64,7 @@ private void OnNewTasks(InvPayload payload) if (!sessions.TryGetValue(Sender, out TaskSession session)) return; // Do not accept payload of type InventoryType.TX if not synced on best known HeaderHeight - if (payload.Type == InventoryType.TX && Blockchain.Singleton.Height < Blockchain.Singleton.HeaderHeight) + if (payload.Type == InventoryType.TX && NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View) < sessions.Values.Max(p => p.LastBlockIndex)) { RequestTasks(session); return; @@ -234,8 +233,7 @@ private void RequestTasks(TaskSession session) { session.AvailableTasks.Remove(knownHashes); // Search any similar hash that is on Singleton's knowledge, which means, on the way or already processed - using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) - session.AvailableTasks.RemoveWhere(p => NativeContract.Ledger.ContainsBlock(snapshot, p)); + session.AvailableTasks.RemoveWhere(p => NativeContract.Ledger.ContainsBlock(Blockchain.Singleton.View, p)); HashSet hashes = new HashSet(session.AvailableTasks); hashes.Remove(MemPoolTaskHash); if (hashes.Count > 0) @@ -254,42 +252,39 @@ private void RequestTasks(TaskSession session) } } + uint headerHeight = Blockchain.Singleton.HeaderHeight(Blockchain.Singleton.View); + uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks - if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && Blockchain.Singleton.HeaderHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCacheFull) + if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCacheFull) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlockByIndexPayload.Create(Blockchain.Singleton.HeaderHeight))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlockByIndexPayload.Create(headerHeight))); } - else if (Blockchain.Singleton.Height < session.LastBlockIndex) + else if (currentHeight < session.LastBlockIndex) { - UInt256 hash = Blockchain.Singleton.CurrentBlockHash; - using (SnapshotCache snapshot = Blockchain.Singleton.GetSnapshot()) + UInt256 hash = NativeContract.Ledger.CurrentHash(Blockchain.Singleton.View); + for (uint i = currentHeight + 1; i <= headerHeight; i++) { - for (uint i = Blockchain.Singleton.Height + 1; i <= Blockchain.Singleton.HeaderHeight; i++) + hash = Blockchain.Singleton.GetBlockHash(i, Blockchain.Singleton.View); + if (!globalTasks.ContainsKey(hash)) { - hash = Blockchain.Singleton.GetBlockHash(i, snapshot); - if (!globalTasks.ContainsKey(hash)) - { - hash = Blockchain.Singleton.GetBlockHash(i - 1, snapshot); - break; - } + hash = Blockchain.Singleton.GetBlockHash(i - 1, Blockchain.Singleton.View); + break; } - node.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(currentHeight))); - session.ExpireTime = TimeProvider.Current.UtcNow.AddMilliseconds(PingCoolingOffPeriod); } session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); } - else if (Blockchain.Singleton.HeaderHeight >= session.LastBlockIndex - && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(Blockchain.Singleton.GetSnapshot(), Blockchain.Singleton.CurrentHeaderHash)?.Timestamp) + else if (headerHeight >= session.LastBlockIndex + && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(Blockchain.Singleton.View, Blockchain.Singleton.CurrentHeaderHash(Blockchain.Singleton.View))?.Timestamp) { if (session.AvailableTasks.Remove(MemPoolTaskHash)) { session.RemoteNode.Tell(Message.Create(MessageCommand.Mempool)); } - session.RemoteNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(Blockchain.Singleton.Height))); + session.RemoteNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(currentHeight))); } } } From 7710ab975eabc26fc97bb18c173e9b4d3388c8bb Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 29 Jan 2021 10:47:38 +0800 Subject: [PATCH 34/91] Update OnGetHeadersMessageReceived --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 085706945d..18a474cb41 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -275,12 +275,12 @@ private void OnGetDataMessageReceived(InvPayload payload) private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) { DataCache snapshot = Blockchain.Singleton.View; - if (payload.IndexStart > NativeContract.Ledger.CurrentIndex(snapshot)) return; + if (payload.IndexStart > Blockchain.Singleton.HeaderHeight(snapshot)) return; List
headers = new List
(); uint count = payload.Count == -1 ? HeadersPayload.MaxHeadersCount : (uint)payload.Count; for (uint i = 0; i < count; i++) { - var header = NativeContract.Ledger.GetHeader(snapshot, payload.IndexStart + i); + var header = Blockchain.Singleton.GetHeader(payload.IndexStart + i, snapshot); if (header == null) break; headers.Add(header); } From 052559aaab5176e475585123c7068c37a475f1b0 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 29 Jan 2021 11:33:28 +0800 Subject: [PATCH 35/91] Add lock for IndexedQueue --- src/neo/IO/Caching/IndexedQueue.cs | 242 +++++++++++++----- src/neo/Ledger/Blockchain.cs | 6 + .../IO/Caching/UT_IndexedQueue.cs | 2 + 3 files changed, 185 insertions(+), 65 deletions(-) diff --git a/src/neo/IO/Caching/IndexedQueue.cs b/src/neo/IO/Caching/IndexedQueue.cs index b01c6ffea9..becb873fd6 100644 --- a/src/neo/IO/Caching/IndexedQueue.cs +++ b/src/neo/IO/Caching/IndexedQueue.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Threading; namespace Neo.IO.Caching { @@ -11,6 +12,7 @@ namespace Neo.IO.Caching /// The type of items in the queue class IndexedQueue : IReadOnlyCollection { + private readonly ReaderWriterLockSlim readerWriterLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); private const int DefaultCapacity = 16; private const int GrowthFactor = 2; private const float TrimThreshold = 0.9f; @@ -22,7 +24,21 @@ class IndexedQueue : IReadOnlyCollection /// /// Indicates the count of items in the queue /// - public int Count => _count; + public int Count + { + get + { + readerWriterLock.EnterReadLock(); + try + { + return _count; + } + finally + { + readerWriterLock.ExitReadLock(); + } + } + } /// /// Creates a queue with the default capacity @@ -55,6 +71,14 @@ public IndexedQueue(IEnumerable collection) _count = _array.Length; } + /// + /// Resource releasing + /// + public void Dispose() + { + readerWriterLock.Dispose(); + } + /// /// Gets the value at the index /// @@ -64,9 +88,17 @@ public ref T this[int index] { get { - if (index < 0 || index >= _count) - throw new IndexOutOfRangeException(); - return ref _array[(index + _head) % _array.Length]; + readerWriterLock.EnterReadLock(); + try + { + if (index < 0 || index >= _count) + throw new IndexOutOfRangeException(); + return ref _array[(index + _head) % _array.Length]; + } + finally + { + readerWriterLock.ExitReadLock(); + } } } @@ -76,24 +108,32 @@ public ref T this[int index] /// The item to insert public void Enqueue(T item) { - if (_array.Length == _count) + readerWriterLock.EnterWriteLock(); + try { - int newSize = _array.Length * GrowthFactor; - if (_head == 0) + if (_array.Length == _count) { - Array.Resize(ref _array, newSize); - } - else - { - T[] buffer = new T[newSize]; - Array.Copy(_array, _head, buffer, 0, _array.Length - _head); - Array.Copy(_array, 0, buffer, _array.Length - _head, _head); - _array = buffer; - _head = 0; + int newSize = _array.Length * GrowthFactor; + if (_head == 0) + { + Array.Resize(ref _array, newSize); + } + else + { + T[] buffer = new T[newSize]; + Array.Copy(_array, _head, buffer, 0, _array.Length - _head); + Array.Copy(_array, 0, buffer, _array.Length - _head, _head); + _array = buffer; + _head = 0; + } } + _array[(_head + _count) % _array.Length] = item; + ++_count; + } + finally + { + readerWriterLock.ExitWriteLock(); } - _array[(_head + _count) % _array.Length] = item; - ++_count; } /// @@ -102,9 +142,17 @@ public void Enqueue(T item) /// The frontmost item public T Peek() { - if (_count == 0) - throw new InvalidOperationException("The queue is empty."); - return _array[_head]; + readerWriterLock.EnterReadLock(); + try + { + if (_count == 0) + throw new InvalidOperationException("The queue is empty."); + return _array[_head]; + } + finally + { + readerWriterLock.ExitReadLock(); + } } /// @@ -114,15 +162,23 @@ public T Peek() /// True if the queue returned an item or false if the queue is empty public bool TryPeek(out T item) { - if (_count == 0) + readerWriterLock.EnterReadLock(); + try { - item = default; - return false; + if (_count == 0) + { + item = default; + return false; + } + else + { + item = _array[_head]; + return true; + } } - else + finally { - item = _array[_head]; - return true; + readerWriterLock.ExitReadLock(); } } @@ -132,13 +188,21 @@ public bool TryPeek(out T item) /// The item that was removed public T Dequeue() { - if (_count == 0) - throw new InvalidOperationException("The queue is empty"); - T result = _array[_head]; - ++_head; - _head %= _array.Length; - --_count; - return result; + readerWriterLock.EnterWriteLock(); + try + { + if (_count == 0) + throw new InvalidOperationException("The queue is empty"); + T result = _array[_head]; + ++_head; + _head %= _array.Length; + --_count; + return result; + } + finally + { + readerWriterLock.ExitWriteLock(); + } } /// @@ -148,18 +212,26 @@ public T Dequeue() /// True if the queue returned an item or false if the queue is empty public bool TryDequeue(out T item) { - if (_count == 0) + readerWriterLock.EnterWriteLock(); + try { - item = default; - return false; + if (_count == 0) + { + item = default; + return false; + } + else + { + item = _array[_head]; + ++_head; + _head %= _array.Length; + --_count; + return true; + } } - else + finally { - item = _array[_head]; - ++_head; - _head %= _array.Length; - --_count; - return true; + readerWriterLock.ExitWriteLock(); } } @@ -168,8 +240,16 @@ public bool TryDequeue(out T item) /// public void Clear() { - _head = 0; - _count = 0; + readerWriterLock.EnterWriteLock(); + try + { + _head = 0; + _count = 0; + } + finally + { + readerWriterLock.ExitWriteLock(); + } } /// @@ -177,16 +257,24 @@ public void Clear() /// public void TrimExcess() { - if (_count == 0) + readerWriterLock.EnterWriteLock(); + try { - _array = new T[DefaultCapacity]; + if (_count == 0) + { + _array = new T[DefaultCapacity]; + } + else if (_array.Length * TrimThreshold >= _count) + { + T[] arr = new T[_count]; + CopyTo(arr, 0); + _array = arr; + _head = 0; + } } - else if (_array.Length * TrimThreshold >= _count) + finally { - T[] arr = new T[_count]; - CopyTo(arr, 0); - _array = arr; - _head = 0; + readerWriterLock.ExitWriteLock(); } } @@ -197,17 +285,25 @@ public void TrimExcess() /// The index in the destination to start copying at public void CopyTo(T[] array, int arrayIndex) { - if (array is null) throw new ArgumentNullException(nameof(array)); - if (arrayIndex < 0 || arrayIndex + _count > array.Length) - throw new ArgumentOutOfRangeException(nameof(arrayIndex)); - if (_head + _count <= _array.Length) + if (readerWriterLock.RecursiveWriteCount == 0) readerWriterLock.EnterReadLock(); + try { - Array.Copy(_array, _head, array, arrayIndex, _count); + if (array is null) throw new ArgumentNullException(nameof(array)); + if (arrayIndex < 0 || arrayIndex + _count > array.Length) + throw new ArgumentOutOfRangeException(nameof(arrayIndex)); + if (_head + _count <= _array.Length) + { + Array.Copy(_array, _head, array, arrayIndex, _count); + } + else + { + Array.Copy(_array, _head, array, arrayIndex, _array.Length - _head); + Array.Copy(_array, 0, array, arrayIndex + _array.Length - _head, _count + _head - _array.Length); + } } - else + finally { - Array.Copy(_array, _head, array, arrayIndex, _array.Length - _head); - Array.Copy(_array, 0, array, arrayIndex + _array.Length - _head, _count + _head - _array.Length); + if (readerWriterLock.RecursiveWriteCount == 0) readerWriterLock.ExitReadLock(); } } @@ -217,15 +313,31 @@ public void CopyTo(T[] array, int arrayIndex) /// An array containing the queue's items public T[] ToArray() { - T[] result = new T[_count]; - CopyTo(result, 0); - return result; + readerWriterLock.EnterReadLock(); + try + { + T[] result = new T[_count]; + CopyTo(result, 0); + return result; + } + finally + { + readerWriterLock.ExitReadLock(); + } } public IEnumerator GetEnumerator() { - for (int i = 0; i < _count; i++) - yield return _array[(_head + i) % _array.Length]; + readerWriterLock.EnterReadLock(); + try + { + for (int i = 0; i < _count; i++) + yield return _array[(_head + i) % _array.Length]; + } + finally + { + readerWriterLock.ExitReadLock(); + } } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 0d5c7367eb..748dc68075 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -438,6 +438,12 @@ private void Persist(Block block) Context.System.EventStream.Publish(new PersistCompleted { Block = block }); } + protected override void PostStop() + { + base.PostStop(); + header_cache.Dispose(); + } + public static Props Props(NeoSystem system, IStore store) { return Akka.Actor.Props.Create(() => new Blockchain(system, store)).WithMailbox("blockchain-mailbox"); diff --git a/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs b/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs index e09726924c..40bb39c47c 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs @@ -34,6 +34,8 @@ public void TestDefault() Assert.ThrowsException(() => _ = queue[1]); Assert.ThrowsException(() => queue[1] = 1); Assert.ThrowsException(() => new IndexedQueue(-1)); + + queue.Dispose(); } [TestMethod] From 33fceba7bd17eaea7616c845861a97eb1744c342 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Fri, 29 Jan 2021 12:28:23 +0800 Subject: [PATCH 36/91] Revert "Add lock for IndexedQueue" This reverts commit 052559aaab5176e475585123c7068c37a475f1b0. --- src/neo/IO/Caching/IndexedQueue.cs | 242 +++++------------- src/neo/Ledger/Blockchain.cs | 6 - .../IO/Caching/UT_IndexedQueue.cs | 2 - 3 files changed, 65 insertions(+), 185 deletions(-) diff --git a/src/neo/IO/Caching/IndexedQueue.cs b/src/neo/IO/Caching/IndexedQueue.cs index becb873fd6..b01c6ffea9 100644 --- a/src/neo/IO/Caching/IndexedQueue.cs +++ b/src/neo/IO/Caching/IndexedQueue.cs @@ -2,7 +2,6 @@ using System.Collections; using System.Collections.Generic; using System.Linq; -using System.Threading; namespace Neo.IO.Caching { @@ -12,7 +11,6 @@ namespace Neo.IO.Caching /// The type of items in the queue class IndexedQueue : IReadOnlyCollection { - private readonly ReaderWriterLockSlim readerWriterLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); private const int DefaultCapacity = 16; private const int GrowthFactor = 2; private const float TrimThreshold = 0.9f; @@ -24,21 +22,7 @@ class IndexedQueue : IReadOnlyCollection /// /// Indicates the count of items in the queue /// - public int Count - { - get - { - readerWriterLock.EnterReadLock(); - try - { - return _count; - } - finally - { - readerWriterLock.ExitReadLock(); - } - } - } + public int Count => _count; /// /// Creates a queue with the default capacity @@ -71,14 +55,6 @@ public IndexedQueue(IEnumerable collection) _count = _array.Length; } - /// - /// Resource releasing - /// - public void Dispose() - { - readerWriterLock.Dispose(); - } - /// /// Gets the value at the index /// @@ -88,17 +64,9 @@ public ref T this[int index] { get { - readerWriterLock.EnterReadLock(); - try - { - if (index < 0 || index >= _count) - throw new IndexOutOfRangeException(); - return ref _array[(index + _head) % _array.Length]; - } - finally - { - readerWriterLock.ExitReadLock(); - } + if (index < 0 || index >= _count) + throw new IndexOutOfRangeException(); + return ref _array[(index + _head) % _array.Length]; } } @@ -108,32 +76,24 @@ public ref T this[int index] /// The item to insert public void Enqueue(T item) { - readerWriterLock.EnterWriteLock(); - try + if (_array.Length == _count) { - if (_array.Length == _count) + int newSize = _array.Length * GrowthFactor; + if (_head == 0) { - int newSize = _array.Length * GrowthFactor; - if (_head == 0) - { - Array.Resize(ref _array, newSize); - } - else - { - T[] buffer = new T[newSize]; - Array.Copy(_array, _head, buffer, 0, _array.Length - _head); - Array.Copy(_array, 0, buffer, _array.Length - _head, _head); - _array = buffer; - _head = 0; - } + Array.Resize(ref _array, newSize); + } + else + { + T[] buffer = new T[newSize]; + Array.Copy(_array, _head, buffer, 0, _array.Length - _head); + Array.Copy(_array, 0, buffer, _array.Length - _head, _head); + _array = buffer; + _head = 0; } - _array[(_head + _count) % _array.Length] = item; - ++_count; - } - finally - { - readerWriterLock.ExitWriteLock(); } + _array[(_head + _count) % _array.Length] = item; + ++_count; } /// @@ -142,17 +102,9 @@ public void Enqueue(T item) /// The frontmost item public T Peek() { - readerWriterLock.EnterReadLock(); - try - { - if (_count == 0) - throw new InvalidOperationException("The queue is empty."); - return _array[_head]; - } - finally - { - readerWriterLock.ExitReadLock(); - } + if (_count == 0) + throw new InvalidOperationException("The queue is empty."); + return _array[_head]; } /// @@ -162,23 +114,15 @@ public T Peek() /// True if the queue returned an item or false if the queue is empty public bool TryPeek(out T item) { - readerWriterLock.EnterReadLock(); - try + if (_count == 0) { - if (_count == 0) - { - item = default; - return false; - } - else - { - item = _array[_head]; - return true; - } + item = default; + return false; } - finally + else { - readerWriterLock.ExitReadLock(); + item = _array[_head]; + return true; } } @@ -188,21 +132,13 @@ public bool TryPeek(out T item) /// The item that was removed public T Dequeue() { - readerWriterLock.EnterWriteLock(); - try - { - if (_count == 0) - throw new InvalidOperationException("The queue is empty"); - T result = _array[_head]; - ++_head; - _head %= _array.Length; - --_count; - return result; - } - finally - { - readerWriterLock.ExitWriteLock(); - } + if (_count == 0) + throw new InvalidOperationException("The queue is empty"); + T result = _array[_head]; + ++_head; + _head %= _array.Length; + --_count; + return result; } /// @@ -212,26 +148,18 @@ public T Dequeue() /// True if the queue returned an item or false if the queue is empty public bool TryDequeue(out T item) { - readerWriterLock.EnterWriteLock(); - try + if (_count == 0) { - if (_count == 0) - { - item = default; - return false; - } - else - { - item = _array[_head]; - ++_head; - _head %= _array.Length; - --_count; - return true; - } + item = default; + return false; } - finally + else { - readerWriterLock.ExitWriteLock(); + item = _array[_head]; + ++_head; + _head %= _array.Length; + --_count; + return true; } } @@ -240,16 +168,8 @@ public bool TryDequeue(out T item) /// public void Clear() { - readerWriterLock.EnterWriteLock(); - try - { - _head = 0; - _count = 0; - } - finally - { - readerWriterLock.ExitWriteLock(); - } + _head = 0; + _count = 0; } /// @@ -257,24 +177,16 @@ public void Clear() /// public void TrimExcess() { - readerWriterLock.EnterWriteLock(); - try + if (_count == 0) { - if (_count == 0) - { - _array = new T[DefaultCapacity]; - } - else if (_array.Length * TrimThreshold >= _count) - { - T[] arr = new T[_count]; - CopyTo(arr, 0); - _array = arr; - _head = 0; - } + _array = new T[DefaultCapacity]; } - finally + else if (_array.Length * TrimThreshold >= _count) { - readerWriterLock.ExitWriteLock(); + T[] arr = new T[_count]; + CopyTo(arr, 0); + _array = arr; + _head = 0; } } @@ -285,25 +197,17 @@ public void TrimExcess() /// The index in the destination to start copying at public void CopyTo(T[] array, int arrayIndex) { - if (readerWriterLock.RecursiveWriteCount == 0) readerWriterLock.EnterReadLock(); - try + if (array is null) throw new ArgumentNullException(nameof(array)); + if (arrayIndex < 0 || arrayIndex + _count > array.Length) + throw new ArgumentOutOfRangeException(nameof(arrayIndex)); + if (_head + _count <= _array.Length) { - if (array is null) throw new ArgumentNullException(nameof(array)); - if (arrayIndex < 0 || arrayIndex + _count > array.Length) - throw new ArgumentOutOfRangeException(nameof(arrayIndex)); - if (_head + _count <= _array.Length) - { - Array.Copy(_array, _head, array, arrayIndex, _count); - } - else - { - Array.Copy(_array, _head, array, arrayIndex, _array.Length - _head); - Array.Copy(_array, 0, array, arrayIndex + _array.Length - _head, _count + _head - _array.Length); - } + Array.Copy(_array, _head, array, arrayIndex, _count); } - finally + else { - if (readerWriterLock.RecursiveWriteCount == 0) readerWriterLock.ExitReadLock(); + Array.Copy(_array, _head, array, arrayIndex, _array.Length - _head); + Array.Copy(_array, 0, array, arrayIndex + _array.Length - _head, _count + _head - _array.Length); } } @@ -313,31 +217,15 @@ public void CopyTo(T[] array, int arrayIndex) /// An array containing the queue's items public T[] ToArray() { - readerWriterLock.EnterReadLock(); - try - { - T[] result = new T[_count]; - CopyTo(result, 0); - return result; - } - finally - { - readerWriterLock.ExitReadLock(); - } + T[] result = new T[_count]; + CopyTo(result, 0); + return result; } public IEnumerator GetEnumerator() { - readerWriterLock.EnterReadLock(); - try - { - for (int i = 0; i < _count; i++) - yield return _array[(_head + i) % _array.Length]; - } - finally - { - readerWriterLock.ExitReadLock(); - } + for (int i = 0; i < _count; i++) + yield return _array[(_head + i) % _array.Length]; } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 748dc68075..0d5c7367eb 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -438,12 +438,6 @@ private void Persist(Block block) Context.System.EventStream.Publish(new PersistCompleted { Block = block }); } - protected override void PostStop() - { - base.PostStop(); - header_cache.Dispose(); - } - public static Props Props(NeoSystem system, IStore store) { return Akka.Actor.Props.Create(() => new Blockchain(system, store)).WithMailbox("blockchain-mailbox"); diff --git a/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs b/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs index 40bb39c47c..e09726924c 100644 --- a/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs +++ b/tests/neo.UnitTests/IO/Caching/UT_IndexedQueue.cs @@ -34,8 +34,6 @@ public void TestDefault() Assert.ThrowsException(() => _ = queue[1]); Assert.ThrowsException(() => queue[1] = 1); Assert.ThrowsException(() => new IndexedQueue(-1)); - - queue.Dispose(); } [TestMethod] From 026c7fedeb5d45ea8e2ff2deac9925f682100ba1 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Fri, 29 Jan 2021 14:08:51 +0800 Subject: [PATCH 37/91] Add HeaderCache and locks --- src/neo/Ledger/Blockchain.cs | 72 ++++++---------- src/neo/Ledger/HeaderCache.cs | 84 +++++++++++++++++++ src/neo/Network/P2P/Payloads/BlockBase.cs | 5 +- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 4 +- src/neo/Network/P2P/TaskManager.cs | 26 +++--- 5 files changed, 130 insertions(+), 61 deletions(-) create mode 100644 src/neo/Ledger/HeaderCache.cs diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 0d5c7367eb..5894ea3a3c 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -62,7 +62,6 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu private static readonly object lockObj = new object(); private readonly NeoSystem system; private readonly IActorRef txrouter; - private readonly IndexedQueue
header_cache = new IndexedQueue
(); private readonly ConcurrentDictionary block_cache = new ConcurrentDictionary(); private readonly Dictionary block_cache_unverified = new Dictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); @@ -75,9 +74,7 @@ public class RelayResult { public IInventory Inventory; public VerifyResult Resu ///
public DataCache View => new SnapshotCache(Store); public MemoryPool MemPool { get; } - public uint HeaderHeight(DataCache snapshot) => header_cache.Count > 0 ? header_cache[^1].Index : NativeContract.Ledger.CurrentIndex(snapshot); - public UInt256 CurrentHeaderHash(DataCache snapshot) => header_cache.Count > 0 ? header_cache[^1].Hash : NativeContract.Ledger.CurrentHash(snapshot); - public bool HeaderCacheFull => header_cache.Count >= 10000; + public HeaderCache HeaderCache { get; } = new HeaderCache(); private static Blockchain singleton; public static Blockchain Singleton @@ -128,6 +125,12 @@ public Blockchain(NeoSystem system, IStore store) } } + protected override void PostStop() + { + base.PostStop(); + HeaderCache.Dispose(); + } + private bool ContainsTransaction(UInt256 hash) { if (MemPool.ContainsKey(hash)) return true; @@ -225,21 +228,22 @@ private VerifyResult OnNewBlock(Block block) { DataCache snapshot = View; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); + uint headerHeight = HeaderCache.Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); if (block.Index <= currentHeight) return VerifyResult.AlreadyExists; - if (block.Index - 1 > HeaderHeight(snapshot)) + if (block.Index - 1 > headerHeight) { AddUnverifiedBlockToCache(block); return VerifyResult.UnableToVerify; } - if (block.Index == HeaderHeight(snapshot) + 1) + if (block.Index == headerHeight + 1) { if (!block.Verify(snapshot)) return VerifyResult.Invalid; } else { - if (!block.Hash.Equals(GetBlockHash(block.Index, snapshot))) + if (!block.Hash.Equals(HeaderCache[block.Index].Hash)) return VerifyResult.Invalid; } block_cache.TryAdd(block.Hash, block); @@ -250,8 +254,8 @@ private VerifyResult OnNewBlock(Block block) while (true) { blocksToPersistList.Add(block_persist); - if (block_persist.Index + 1 > HeaderHeight(snapshot)) break; - UInt256 hash = GetCachedHeader(block_persist.Index + 1).Hash; + if (block_persist.Index + 1 > headerHeight) break; + UInt256 hash = HeaderCache[block_persist.Index + 1].Hash; if (!block_cache.TryGetValue(hash, out block_persist)) break; } @@ -268,7 +272,7 @@ private VerifyResult OnNewBlock(Block block) // Empirically calibrated for relaying the most recent 2 blocks persisted with 15s network // Increase in the rate of 1 block per second in configurations with faster blocks - if (blockToPersist.Index + 99 >= HeaderHeight(snapshot)) + if (blockToPersist.Index + 99 >= headerHeight) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = blockToPersist }); } if (block_cache_unverified.TryGetValue(currentHeight + 1, out var unverifiedBlocks)) @@ -282,26 +286,26 @@ private VerifyResult OnNewBlock(Block block) } else { - if (block.Index + 99 >= HeaderHeight(snapshot)) + if (block.Index + 99 >= headerHeight) system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = block }); - if (block.Index == HeaderHeight(snapshot) + 1) - header_cache.Enqueue(block.Header); + if (block.Index == headerHeight + 1) + HeaderCache.Add(block.Header); } return VerifyResult.Succeed; } private void OnNewHeaders(Header[] headers) { - if (HeaderCacheFull) return; - using (SnapshotCache snapshot = GetSnapshot()) + if (HeaderCache.Full) return; + DataCache snapshot = View; + uint headerHeight = HeaderCache.Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); + foreach (Header header in headers) { - foreach (Header header in headers) - { - if (header.Index > HeaderHeight(snapshot) + 1) break; - if (header.Index < HeaderHeight(snapshot) + 1) continue; - if (!header.Verify(snapshot)) break; - header_cache.Enqueue(header); - } + if (header.Index > headerHeight + 1) break; + if (header.Index < headerHeight + 1) continue; + if (!header.Verify(snapshot)) break; + HeaderCache.Add(header); + ++headerHeight; } system.TaskManager.Tell(new TaskManager.HeaderTaskCompleted(), Sender); } @@ -371,7 +375,7 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { - header_cache.TryDequeue(out _); + HeaderCache.TryRemoveFirst(); List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) { @@ -454,28 +458,6 @@ private void SendRelayResult(IInventory inventory, VerifyResult result) Context.System.EventStream.Publish(rr); } - private Header GetCachedHeader(uint index) - { - if (header_cache.Count == 0) return null; - uint firstIndex = header_cache[0].Index; - if (index < firstIndex) return null; - index -= firstIndex; - if (index >= header_cache.Count) return null; - return header_cache[(int)index]; - } - - public Header GetHeader(uint index, DataCache snapshot) - { - Header header = GetCachedHeader(index); - return header != null ? header : NativeContract.Ledger.GetHeader(snapshot, index); - } - - public UInt256 GetBlockHash(uint index, DataCache snapshot) - { - UInt256 hash = GetCachedHeader(index)?.Hash; - return hash != null ? hash : NativeContract.Ledger.GetBlockHash(snapshot, index); - } - private void UpdateExtensibleWitnessWhiteList(DataCache snapshot) { uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs new file mode 100644 index 0000000000..69a21cced2 --- /dev/null +++ b/src/neo/Ledger/HeaderCache.cs @@ -0,0 +1,84 @@ +using Neo.IO.Caching; +using Neo.Network.P2P.Payloads; +using System; +using System.Threading; + +namespace Neo.Ledger +{ + public sealed class HeaderCache : IDisposable + { + private readonly IndexedQueue
headers = new IndexedQueue
(); + private readonly ReaderWriterLockSlim readerWriterLock = new ReaderWriterLockSlim(); + + public Header this[uint index] + { + get + { + readerWriterLock.EnterReadLock(); + try + { + if (headers.Count == 0) return null; + uint firstIndex = headers[0].Index; + if (index < firstIndex) return null; + index -= firstIndex; + if (index >= headers.Count) return null; + return headers[(int)index]; + } + finally + { + readerWriterLock.ExitReadLock(); + } + } + } + + public bool Full => headers.Count >= 10000; + + public Header Last + { + get + { + readerWriterLock.EnterReadLock(); + try + { + if (headers.Count == 0) return null; + return headers[^1]; + } + finally + { + readerWriterLock.ExitReadLock(); + } + } + } + + public void Dispose() + { + readerWriterLock.Dispose(); + } + + internal void Add(Header header) + { + readerWriterLock.EnterWriteLock(); + try + { + headers.Enqueue(header); + } + finally + { + readerWriterLock.ExitWriteLock(); + } + } + + internal void TryRemoveFirst() + { + readerWriterLock.EnterWriteLock(); + try + { + headers.TryDequeue(out _); + } + finally + { + readerWriterLock.ExitWriteLock(); + } + } + } +} diff --git a/src/neo/Network/P2P/Payloads/BlockBase.cs b/src/neo/Network/P2P/Payloads/BlockBase.cs index 14233238b5..517303095b 100644 --- a/src/neo/Network/P2P/Payloads/BlockBase.cs +++ b/src/neo/Network/P2P/Payloads/BlockBase.cs @@ -3,6 +3,7 @@ using Neo.Ledger; using Neo.Persistence; using Neo.SmartContract; +using Neo.SmartContract.Native; using Neo.Wallets; using System; using System.IO; @@ -76,7 +77,7 @@ void IVerifiable.DeserializeUnsigned(BinaryReader reader) UInt160[] IVerifiable.GetScriptHashesForVerifying(DataCache snapshot) { if (PrevHash == UInt256.Zero) return new[] { Witness.ScriptHash }; - var prev = Blockchain.Singleton.GetHeader(Index - 1, snapshot); + var prev = Blockchain.Singleton.HeaderCache[Index - 1] ?? NativeContract.Ledger.GetHeader(snapshot, Index - 1); if (prev is null) throw new InvalidOperationException(); if (prev.Hash != PrevHash) throw new InvalidOperationException(); return new[] { prev.NextConsensus }; @@ -115,7 +116,7 @@ public virtual JObject ToJson() public virtual bool Verify(DataCache snapshot) { - var prev = Blockchain.Singleton.GetHeader(Index - 1, snapshot); + var prev = Blockchain.Singleton.HeaderCache[Index - 1] ?? NativeContract.Ledger.GetHeader(snapshot, Index - 1); if (prev is null) return false; if (prev.Index + 1 != Index) return false; if (prev.Hash != PrevHash) return false; diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 18a474cb41..085706945d 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -275,12 +275,12 @@ private void OnGetDataMessageReceived(InvPayload payload) private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) { DataCache snapshot = Blockchain.Singleton.View; - if (payload.IndexStart > Blockchain.Singleton.HeaderHeight(snapshot)) return; + if (payload.IndexStart > NativeContract.Ledger.CurrentIndex(snapshot)) return; List
headers = new List
(); uint count = payload.Count == -1 ? HeadersPayload.MaxHeadersCount : (uint)payload.Count; for (uint i = 0; i < count; i++) { - var header = Blockchain.Singleton.GetHeader(payload.IndexStart + i, snapshot); + var header = NativeContract.Ledger.GetHeader(snapshot, payload.IndexStart + i); if (header == null) break; headers.Add(header); } diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 2812da956e..768fc6c659 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -228,12 +228,15 @@ public static Props Props(NeoSystem system) private void RequestTasks(TaskSession session) { if (session.HasTask) return; + + DataCache snapshot = Blockchain.Singleton.View; + // If there are pending tasks of InventoryType.Block we should process them if (session.AvailableTasks.Count > 0) { session.AvailableTasks.Remove(knownHashes); // Search any similar hash that is on Singleton's knowledge, which means, on the way or already processed - session.AvailableTasks.RemoveWhere(p => NativeContract.Ledger.ContainsBlock(Blockchain.Singleton.View, p)); + session.AvailableTasks.RemoveWhere(p => NativeContract.Ledger.ContainsBlock(snapshot, p)); HashSet hashes = new HashSet(session.AvailableTasks); hashes.Remove(MemPoolTaskHash); if (hashes.Count > 0) @@ -252,11 +255,13 @@ private void RequestTasks(TaskSession session) } } - uint headerHeight = Blockchain.Singleton.HeaderHeight(Blockchain.Singleton.View); - uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); + Header last = Blockchain.Singleton.HeaderCache.Last; + uint headerHeight = last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); + UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); + uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks - if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCacheFull) + if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); @@ -264,20 +269,17 @@ private void RequestTasks(TaskSession session) } else if (currentHeight < session.LastBlockIndex) { - UInt256 hash = NativeContract.Ledger.CurrentHash(Blockchain.Singleton.View); + UInt256 hash = NativeContract.Ledger.CurrentHash(snapshot); for (uint i = currentHeight + 1; i <= headerHeight; i++) { - hash = Blockchain.Singleton.GetBlockHash(i, Blockchain.Singleton.View); - if (!globalTasks.ContainsKey(hash)) - { - hash = Blockchain.Singleton.GetBlockHash(i - 1, Blockchain.Singleton.View); - break; - } + UInt256 nextHash = Blockchain.Singleton.HeaderCache[i].Hash; + if (!globalTasks.ContainsKey(nextHash)) break; + hash = nextHash; } session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); } else if (headerHeight >= session.LastBlockIndex - && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(Blockchain.Singleton.View, Blockchain.Singleton.CurrentHeaderHash(Blockchain.Singleton.View))?.Timestamp) + && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(snapshot, headerHash)?.Timestamp) { if (session.AvailableTasks.Remove(MemPoolTaskHash)) { From 0e061276783faec4e0937fe8ef99f0569d29cd46 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Fri, 29 Jan 2021 14:14:45 +0800 Subject: [PATCH 38/91] Optimize --- src/neo/Network/P2P/Payloads/BlockBase.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/Payloads/BlockBase.cs b/src/neo/Network/P2P/Payloads/BlockBase.cs index 517303095b..1c0215fa44 100644 --- a/src/neo/Network/P2P/Payloads/BlockBase.cs +++ b/src/neo/Network/P2P/Payloads/BlockBase.cs @@ -77,9 +77,8 @@ void IVerifiable.DeserializeUnsigned(BinaryReader reader) UInt160[] IVerifiable.GetScriptHashesForVerifying(DataCache snapshot) { if (PrevHash == UInt256.Zero) return new[] { Witness.ScriptHash }; - var prev = Blockchain.Singleton.HeaderCache[Index - 1] ?? NativeContract.Ledger.GetHeader(snapshot, Index - 1); + BlockBase prev = Blockchain.Singleton.HeaderCache[Index - 1] ?? (BlockBase)NativeContract.Ledger.GetTrimmedBlock(snapshot, PrevHash); if (prev is null) throw new InvalidOperationException(); - if (prev.Hash != PrevHash) throw new InvalidOperationException(); return new[] { prev.NextConsensus }; } From a4240cf5f5fae2159659bff1685c11bd39819d2f Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Fri, 29 Jan 2021 14:16:43 +0800 Subject: [PATCH 39/91] Optimize --- src/neo/Network/P2P/Payloads/BlockBase.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/neo/Network/P2P/Payloads/BlockBase.cs b/src/neo/Network/P2P/Payloads/BlockBase.cs index 1c0215fa44..a073858cf6 100644 --- a/src/neo/Network/P2P/Payloads/BlockBase.cs +++ b/src/neo/Network/P2P/Payloads/BlockBase.cs @@ -117,7 +117,6 @@ public virtual bool Verify(DataCache snapshot) { var prev = Blockchain.Singleton.HeaderCache[Index - 1] ?? NativeContract.Ledger.GetHeader(snapshot, Index - 1); if (prev is null) return false; - if (prev.Index + 1 != Index) return false; if (prev.Hash != PrevHash) return false; if (prev.Timestamp >= Timestamp) return false; if (!this.VerifyWitnesses(snapshot, 1_00000000)) return false; From 234ffcb42c1f72bde5ffc3f4e48826332130f92e Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 29 Jan 2021 15:17:46 +0800 Subject: [PATCH 40/91] assign GetBlockByIndexPayload to GetBlocks --- src/neo/Network/P2P/MessageCommand.cs | 2 +- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 16 ++++++++-------- src/neo/Network/P2P/TaskManager.cs | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index c66c15799a..bbd12111b4 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -24,7 +24,7 @@ public enum MessageCommand : byte GetHeaders = 0x20, [ReflectionCache(typeof(HeadersPayload))] Headers = 0x21, - [ReflectionCache(typeof(GetBlocksPayload))] + [ReflectionCache(typeof(GetBlockByIndexPayload))] GetBlocks = 0x24, Mempool = 0x25, [ReflectionCache(typeof(InvPayload))] diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 085706945d..6337c1e32c 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -79,7 +79,7 @@ private void OnMessage(Message msg) OnGetAddrMessageReceived(); break; case MessageCommand.GetBlocks: - OnGetBlocksMessageReceived((GetBlocksPayload)msg.Payload); + OnGetBlocksMessageReceived((GetBlockByIndexPayload)msg.Payload); break; case MessageCommand.GetBlockByIndex: OnGetBlockByIndexMessageReceived((GetBlockByIndexPayload)msg.Payload); @@ -166,23 +166,23 @@ private void OnGetAddrMessageReceived() /// /// Will be triggered when a MessageCommand.GetBlocks message is received. - /// Tell the specified number of blocks' hashes starting with the requested HashStart until payload.Count or MaxHashesCount + /// Tell the specified number of blocks' hashes starting with the requested height until payload.Count or MaxHashesCount /// Responses are sent to RemoteNode actor as MessageCommand.Inv Message. /// - /// A GetBlocksPayload including start block Hash and number of blocks requested. - private void OnGetBlocksMessageReceived(GetBlocksPayload payload) + /// A GetBlockByIndexPayload including start block height and number of blocks requested. + private void OnGetBlocksMessageReceived(GetBlockByIndexPayload payload) { // The default value of payload.Count is -1 int count = payload.Count < 0 || payload.Count > InvPayload.MaxHashesCount ? InvPayload.MaxHashesCount : payload.Count; DataCache snapshot = Blockchain.Singleton.View; - UInt256 hash = payload.HashStart; - TrimmedBlock state = NativeContract.Ledger.GetTrimmedBlock(snapshot, hash); - if (state == null) return; + uint startIndex = payload.IndexStart; + UInt256 hash = NativeContract.Ledger.GetBlockHash(snapshot, startIndex); + if (hash == null) return; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); List hashes = new List(); for (uint i = 1; i <= count; i++) { - uint index = state.Index + i; + uint index = startIndex + i; if (index > currentHeight) break; hash = NativeContract.Ledger.GetBlockHash(snapshot, index); diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 768fc6c659..373626a718 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -256,9 +256,9 @@ private void RequestTasks(TaskSession session) } Header last = Blockchain.Singleton.HeaderCache.Last; - uint headerHeight = last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); - UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); + uint headerHeight = last?.Index ?? currentHeight; + UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) @@ -269,14 +269,14 @@ private void RequestTasks(TaskSession session) } else if (currentHeight < session.LastBlockIndex) { - UInt256 hash = NativeContract.Ledger.CurrentHash(snapshot); + uint startHeight = currentHeight; for (uint i = currentHeight + 1; i <= headerHeight; i++) { UInt256 nextHash = Blockchain.Singleton.HeaderCache[i].Hash; if (!globalTasks.ContainsKey(nextHash)) break; - hash = nextHash; + startHeight = i; } - session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlockByIndexPayload.Create(startHeight))); } else if (headerHeight >= session.LastBlockIndex && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(snapshot, headerHash)?.Timestamp) From 35fd13c8648ee5c5a6b1c6e6f0d8577310ada04a Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 29 Jan 2021 10:52:42 +0100 Subject: [PATCH 41/91] Optimize --- src/neo/Ledger/Blockchain.cs | 4 ++-- src/neo/Ledger/HeaderCache.cs | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 5894ea3a3c..47dd5ec40c 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -228,7 +228,7 @@ private VerifyResult OnNewBlock(Block block) { DataCache snapshot = View; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); - uint headerHeight = HeaderCache.Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); + uint headerHeight = HeaderCache.GetCurrentIndex(snapshot); if (block.Index <= currentHeight) return VerifyResult.AlreadyExists; if (block.Index - 1 > headerHeight) @@ -298,7 +298,7 @@ private void OnNewHeaders(Header[] headers) { if (HeaderCache.Full) return; DataCache snapshot = View; - uint headerHeight = HeaderCache.Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); + uint headerHeight = HeaderCache.GetCurrentIndex(snapshot); foreach (Header header in headers) { if (header.Index > headerHeight + 1) break; diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index 69a21cced2..f367f7c48b 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -1,5 +1,7 @@ using Neo.IO.Caching; using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.SmartContract.Native; using System; using System.Threading; @@ -80,5 +82,10 @@ internal void TryRemoveFirst() readerWriterLock.ExitWriteLock(); } } + + internal uint GetCurrentIndex(DataCache snapshot) + { + return Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); + } } } From f0ed92ab315aaf81f6c347c46b32d5d87d0108e4 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 29 Jan 2021 18:30:46 +0800 Subject: [PATCH 42/91] fix sync logic --- src/neo/Network/P2P/TaskManager.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 373626a718..262f28b2b9 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -63,8 +63,10 @@ private void OnNewTasks(InvPayload payload) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; - // Do not accept payload of type InventoryType.TX if not synced on best known HeaderHeight - if (payload.Type == InventoryType.TX && NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View) < sessions.Values.Max(p => p.LastBlockIndex)) + // Do not accept payload of type InventoryType.TX if not synced on HeaderHeight + uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); + uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; + if (payload.Type == InventoryType.TX && currentHeight < headerHeight) { RequestTasks(session); return; From 3c3b43b1f07b67369030921f62d19556d8417de6 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Fri, 29 Jan 2021 20:30:55 +0800 Subject: [PATCH 43/91] Revert "Optimize" This reverts commit 35fd13c8648ee5c5a6b1c6e6f0d8577310ada04a. --- src/neo/Ledger/Blockchain.cs | 4 ++-- src/neo/Ledger/HeaderCache.cs | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 47dd5ec40c..5894ea3a3c 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -228,7 +228,7 @@ private VerifyResult OnNewBlock(Block block) { DataCache snapshot = View; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); - uint headerHeight = HeaderCache.GetCurrentIndex(snapshot); + uint headerHeight = HeaderCache.Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); if (block.Index <= currentHeight) return VerifyResult.AlreadyExists; if (block.Index - 1 > headerHeight) @@ -298,7 +298,7 @@ private void OnNewHeaders(Header[] headers) { if (HeaderCache.Full) return; DataCache snapshot = View; - uint headerHeight = HeaderCache.GetCurrentIndex(snapshot); + uint headerHeight = HeaderCache.Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); foreach (Header header in headers) { if (header.Index > headerHeight + 1) break; diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index f367f7c48b..69a21cced2 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -1,7 +1,5 @@ using Neo.IO.Caching; using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.SmartContract.Native; using System; using System.Threading; @@ -82,10 +80,5 @@ internal void TryRemoveFirst() readerWriterLock.ExitWriteLock(); } } - - internal uint GetCurrentIndex(DataCache snapshot) - { - return Last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); - } } } From 2131befca2360947840c9a5b5cf6168498876427 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 29 Jan 2021 21:06:16 +0800 Subject: [PATCH 44/91] Revert "assign GetBlockByIndexPayload to GetBlocks" This reverts commit 234ffcb42c1f72bde5ffc3f4e48826332130f92e. --- src/neo/Network/P2P/MessageCommand.cs | 2 +- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 16 ++++++++-------- src/neo/Network/P2P/TaskManager.cs | 10 +++++----- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/neo/Network/P2P/MessageCommand.cs b/src/neo/Network/P2P/MessageCommand.cs index bbd12111b4..c66c15799a 100644 --- a/src/neo/Network/P2P/MessageCommand.cs +++ b/src/neo/Network/P2P/MessageCommand.cs @@ -24,7 +24,7 @@ public enum MessageCommand : byte GetHeaders = 0x20, [ReflectionCache(typeof(HeadersPayload))] Headers = 0x21, - [ReflectionCache(typeof(GetBlockByIndexPayload))] + [ReflectionCache(typeof(GetBlocksPayload))] GetBlocks = 0x24, Mempool = 0x25, [ReflectionCache(typeof(InvPayload))] diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 6337c1e32c..085706945d 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -79,7 +79,7 @@ private void OnMessage(Message msg) OnGetAddrMessageReceived(); break; case MessageCommand.GetBlocks: - OnGetBlocksMessageReceived((GetBlockByIndexPayload)msg.Payload); + OnGetBlocksMessageReceived((GetBlocksPayload)msg.Payload); break; case MessageCommand.GetBlockByIndex: OnGetBlockByIndexMessageReceived((GetBlockByIndexPayload)msg.Payload); @@ -166,23 +166,23 @@ private void OnGetAddrMessageReceived() /// /// Will be triggered when a MessageCommand.GetBlocks message is received. - /// Tell the specified number of blocks' hashes starting with the requested height until payload.Count or MaxHashesCount + /// Tell the specified number of blocks' hashes starting with the requested HashStart until payload.Count or MaxHashesCount /// Responses are sent to RemoteNode actor as MessageCommand.Inv Message. /// - /// A GetBlockByIndexPayload including start block height and number of blocks requested. - private void OnGetBlocksMessageReceived(GetBlockByIndexPayload payload) + /// A GetBlocksPayload including start block Hash and number of blocks requested. + private void OnGetBlocksMessageReceived(GetBlocksPayload payload) { // The default value of payload.Count is -1 int count = payload.Count < 0 || payload.Count > InvPayload.MaxHashesCount ? InvPayload.MaxHashesCount : payload.Count; DataCache snapshot = Blockchain.Singleton.View; - uint startIndex = payload.IndexStart; - UInt256 hash = NativeContract.Ledger.GetBlockHash(snapshot, startIndex); - if (hash == null) return; + UInt256 hash = payload.HashStart; + TrimmedBlock state = NativeContract.Ledger.GetTrimmedBlock(snapshot, hash); + if (state == null) return; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); List hashes = new List(); for (uint i = 1; i <= count; i++) { - uint index = startIndex + i; + uint index = state.Index + i; if (index > currentHeight) break; hash = NativeContract.Ledger.GetBlockHash(snapshot, index); diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 262f28b2b9..a87f7123a4 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -258,9 +258,9 @@ private void RequestTasks(TaskSession session) } Header last = Blockchain.Singleton.HeaderCache.Last; - uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); - uint headerHeight = last?.Index ?? currentHeight; + uint headerHeight = last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); + uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) @@ -271,14 +271,14 @@ private void RequestTasks(TaskSession session) } else if (currentHeight < session.LastBlockIndex) { - uint startHeight = currentHeight; + UInt256 hash = NativeContract.Ledger.CurrentHash(snapshot); for (uint i = currentHeight + 1; i <= headerHeight; i++) { UInt256 nextHash = Blockchain.Singleton.HeaderCache[i].Hash; if (!globalTasks.ContainsKey(nextHash)) break; - startHeight = i; + hash = nextHash; } - session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlockByIndexPayload.Create(startHeight))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); } else if (headerHeight >= session.LastBlockIndex && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(snapshot, headerHash)?.Timestamp) From 9e1ff37a46e864d30abaaa2e1926718ed29a3e9a Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Fri, 29 Jan 2021 21:44:08 +0800 Subject: [PATCH 45/91] fix --- src/neo/Network/P2P/TaskManager.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index a87f7123a4..8d01f6087b 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -258,9 +258,9 @@ private void RequestTasks(TaskSession session) } Header last = Blockchain.Singleton.HeaderCache.Last; - uint headerHeight = last?.Index ?? NativeContract.Ledger.CurrentIndex(snapshot); - UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); + uint headerHeight = last?.Index ?? currentHeight; + UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) @@ -271,14 +271,14 @@ private void RequestTasks(TaskSession session) } else if (currentHeight < session.LastBlockIndex) { - UInt256 hash = NativeContract.Ledger.CurrentHash(snapshot); + uint startHeight = currentHeight; for (uint i = currentHeight + 1; i <= headerHeight; i++) { UInt256 nextHash = Blockchain.Singleton.HeaderCache[i].Hash; if (!globalTasks.ContainsKey(nextHash)) break; - hash = nextHash; + startHeight = i; } - session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlocks, GetBlocksPayload.Create(hash))); + session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); } else if (headerHeight >= session.LastBlockIndex && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(snapshot, headerHash)?.Timestamp) From cea69bbaedcab32acb2f0f7e9e6c056ea3bf1a9b Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 29 Jan 2021 15:05:18 +0100 Subject: [PATCH 46/91] Add check null --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 8d01f6087b..2f22599853 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -275,7 +275,7 @@ private void RequestTasks(TaskSession session) for (uint i = currentHeight + 1; i <= headerHeight; i++) { UInt256 nextHash = Blockchain.Singleton.HeaderCache[i].Hash; - if (!globalTasks.ContainsKey(nextHash)) break; + if (nextHash is null || !globalTasks.ContainsKey(nextHash)) break; startHeight = i; } session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); From 8aba54fbfe0e3edd1b5f89b0f087717717c3664d Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 29 Jan 2021 15:06:00 +0100 Subject: [PATCH 47/91] Add nullable --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 2f22599853..ff9b7e9acc 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -274,7 +274,7 @@ private void RequestTasks(TaskSession session) uint startHeight = currentHeight; for (uint i = currentHeight + 1; i <= headerHeight; i++) { - UInt256 nextHash = Blockchain.Singleton.HeaderCache[i].Hash; + UInt256 nextHash = Blockchain.Singleton.HeaderCache[i]?.Hash; if (nextHash is null || !globalTasks.ContainsKey(nextHash)) break; startHeight = i; } From 421317e8bc743801939c784597f2374e4c295fa3 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 03:14:23 +0800 Subject: [PATCH 48/91] Add HeaderCache.GetSnapshot() --- src/neo/Ledger/HeaderCache.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index 69a21cced2..5a39336e35 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -80,5 +80,18 @@ internal void TryRemoveFirst() readerWriterLock.ExitWriteLock(); } } + + public Header[] GetSnapshot() + { + readerWriterLock.EnterReadLock(); + try + { + return headers.ToArray(); + } + finally + { + readerWriterLock.ExitReadLock(); + } + } } } From 1bd4bbbde780c2a03da5ed6db8d6b01262226ccb Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Sat, 30 Jan 2021 10:56:09 +0800 Subject: [PATCH 49/91] fix null pointer --- src/neo/Network/P2P/TaskManager.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index ff9b7e9acc..f0afe0e34d 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -257,7 +257,8 @@ private void RequestTasks(TaskSession session) } } - Header last = Blockchain.Singleton.HeaderCache.Last; + Header[] headers = Blockchain.Singleton.HeaderCache.GetSnapshot(); + Header last = headers == null ? null : (headers.Length > 0 ? headers[headers.Length - 1] : null); uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); uint headerHeight = last?.Index ?? currentHeight; UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); @@ -272,10 +273,11 @@ private void RequestTasks(TaskSession session) else if (currentHeight < session.LastBlockIndex) { uint startHeight = currentHeight; - for (uint i = currentHeight + 1; i <= headerHeight; i++) + for (uint i = 0; i < headers.Length; i++) { - UInt256 nextHash = Blockchain.Singleton.HeaderCache[i]?.Hash; - if (nextHash is null || !globalTasks.ContainsKey(nextHash)) break; + if (headers[i].Index <= currentHeight) continue; + UInt256 nextHash = headers[i].Hash; + if (!globalTasks.ContainsKey(nextHash)) break; startHeight = i; } session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); From 51fc9ecea091a60eba9d2e82a2e15d604c0a5fa1 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 16:41:46 +0800 Subject: [PATCH 50/91] Optimize --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index f0afe0e34d..a9f9b80616 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -258,7 +258,7 @@ private void RequestTasks(TaskSession session) } Header[] headers = Blockchain.Singleton.HeaderCache.GetSnapshot(); - Header last = headers == null ? null : (headers.Length > 0 ? headers[headers.Length - 1] : null); + Header last = headers.Length > 0 ? headers[^1] : null; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); uint headerHeight = last?.Index ?? currentHeight; UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); From 7f1a36287db9d5f21ed46182fa256e5a9ecac2c1 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 17:25:47 +0800 Subject: [PATCH 51/91] Do not send ping when persisted block --- src/neo/Ledger/Blockchain.cs | 2 -- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 5894ea3a3c..bbea1609b6 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -281,8 +281,6 @@ private VerifyResult OnNewBlock(Block block) Self.Tell(unverifiedBlock, ActorRefs.NoSender); block_cache_unverified.Remove(block.Index + 1); } - // We can store the new block in block_cache and tell the new height to other nodes after Persist(). - system.LocalNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(block.Index))); } else { diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 085706945d..1558f49762 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -290,6 +290,7 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) private void OnHeadersMessageReceived(HeadersPayload payload) { + UpdateLastBlockIndex(payload.Headers[^1].Index); system.Blockchain.Tell(payload.Headers); } From 6e1f5976938b3f22a4629b486db17055be936ae4 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 17:29:09 +0800 Subject: [PATCH 52/91] Remove TaskCompleted --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 2 +- src/neo/Network/P2P/TaskManager.cs | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 1558f49762..9db03c53dd 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -303,7 +303,7 @@ private void OnInventoryReceived(IInventory inventory) UpdateLastBlockIndex(block.Index); } knownHashes.Add(inventory.Hash); - system.TaskManager.Tell(new TaskManager.TaskCompleted { Hash = inventory.Hash }); + system.TaskManager.Tell(inventory); system.Blockchain.Tell(inventory); } diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index a9f9b80616..c5542b3c82 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -19,7 +19,6 @@ public class TaskManager : UntypedActor internal class Register { public VersionPayload Version; } internal class Update { public uint LastBlockIndex; } internal class NewTasks { public InvPayload Payload; } - public class TaskCompleted { public UInt256 Hash; } public class HeaderTaskCompleted { } public class RestartTasks { public InvPayload Payload; } private class Timer { } @@ -110,15 +109,15 @@ protected override void OnReceive(object message) case NewTasks tasks: OnNewTasks(tasks.Payload); break; - case TaskCompleted completed: - OnTaskCompleted(completed.Hash); - break; case HeaderTaskCompleted _: OnHeaderTaskCompleted(); break; case RestartTasks restart: OnRestartTasks(restart.Payload); break; + case IInventory inventory: + OnTaskCompleted(inventory.Hash); + break; case Timer _: OnTimer(); break; From 0810e8026bc84adb66f1c81d07a3bb5213987e22 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 17:42:02 +0800 Subject: [PATCH 53/91] Remove HeaderTaskCompleted --- src/neo/Ledger/Blockchain.cs | 1 - src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 1 + src/neo/Network/P2P/TaskManager.cs | 9 ++++----- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index bbea1609b6..5c872a4f2f 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -305,7 +305,6 @@ private void OnNewHeaders(Header[] headers) HeaderCache.Add(header); ++headerHeight; } - system.TaskManager.Tell(new TaskManager.HeaderTaskCompleted(), Sender); } private VerifyResult OnNewInventory(IInventory inventory) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 9db03c53dd..21d095c758 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -291,6 +291,7 @@ private void OnGetHeadersMessageReceived(GetBlockByIndexPayload payload) private void OnHeadersMessageReceived(HeadersPayload payload) { UpdateLastBlockIndex(payload.Headers[^1].Index); + system.TaskManager.Tell(payload.Headers); system.Blockchain.Tell(payload.Headers); } diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c5542b3c82..76acc97b5f 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -19,7 +19,6 @@ public class TaskManager : UntypedActor internal class Register { public VersionPayload Version; } internal class Update { public uint LastBlockIndex; } internal class NewTasks { public InvPayload Payload; } - public class HeaderTaskCompleted { } public class RestartTasks { public InvPayload Payload; } private class Timer { } @@ -49,7 +48,7 @@ public TaskManager(NeoSystem system) Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } - private void OnHeaderTaskCompleted() + private void OnHeaders(Header[] headers) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; @@ -109,12 +108,12 @@ protected override void OnReceive(object message) case NewTasks tasks: OnNewTasks(tasks.Payload); break; - case HeaderTaskCompleted _: - OnHeaderTaskCompleted(); - break; case RestartTasks restart: OnRestartTasks(restart.Payload); break; + case Header[] headers: + OnHeaders(headers); + break; case IInventory inventory: OnTaskCompleted(inventory.Hash); break; From 131f2df9f885c3e56356752d44dfb7d7c3aeb06d Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Sat, 30 Jan 2021 18:04:27 +0800 Subject: [PATCH 54/91] restore OnBlock --- src/neo/Network/P2P/TaskManager.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 76acc97b5f..e20cc36e78 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -48,7 +48,14 @@ public TaskManager(NeoSystem system) Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } - private void OnHeaders(Header[] headers) + private void OnBlock(Block _) + { + if (!sessions.TryGetValue(Sender, out TaskSession session)) + return; + RequestTasks(session); + } + + private void OnHeaders(Header[] _) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; @@ -111,6 +118,9 @@ protected override void OnReceive(object message) case RestartTasks restart: OnRestartTasks(restart.Payload); break; + case Block block: + OnBlock(block); + break; case Header[] headers: OnHeaders(headers); break; From 44781e10be50ffc6bd94bbe33c982d863ef7b01e Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 18:37:35 +0800 Subject: [PATCH 55/91] Optimize ping --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 10 +++++----- src/neo/Network/P2P/RemoteNode.cs | 7 ++++++- src/neo/Network/P2P/TaskManager.cs | 2 -- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 21d095c758..8f3936550a 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -34,8 +34,6 @@ protected override UInt256 GetKeyForItem((UInt256, DateTime) item) private BloomFilter bloom_filter; private static readonly TimeSpan TimerInterval = TimeSpan.FromSeconds(30); - private static readonly TimeSpan PendingTimeout = TimeSpan.FromMinutes(1); - private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); private void OnMessage(Message msg) @@ -382,15 +380,17 @@ private void OnVersionMessageReceived(VersionPayload payload) SendMessage(Message.Create(MessageCommand.Verack)); } - private void RefreshPendingKnownHashes() + private void OnTimer() { + DateTime oneMinuteAgo = TimeProvider.Current.UtcNow.AddMinutes(-1); while (pendingKnownHashes.Count > 0) { var (_, time) = pendingKnownHashes[0]; - if (TimeProvider.Current.UtcNow - time <= PendingTimeout) - break; + if (oneMinuteAgo <= time) break; pendingKnownHashes.RemoveAt(0); } + if (oneMinuteAgo > lastSent) + EnqueueMessage(Message.Create(MessageCommand.Ping, PingPayload.Create(NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View)))); } private void UpdateLastBlockIndex(uint lastBlockIndex) diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index a417e4115e..dee2b893f3 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -8,6 +8,7 @@ using Neo.Network.P2P.Capabilities; using Neo.Network.P2P.Payloads; using Neo.SmartContract.Native; +using System; using System.Collections; using System.Collections.Generic; using System.Linq; @@ -23,6 +24,7 @@ internal class Relay { public IInventory Inventory; } private readonly NeoSystem system; private readonly Queue message_queue_high = new Queue(); private readonly Queue message_queue_low = new Queue(); + private DateTime lastSent = TimeProvider.Current.UtcNow; private readonly bool[] sentCommands = new bool[1 << (sizeof(MessageCommand) * 8)]; private ByteString msg_buffer = ByteString.Empty; private bool ack = true; @@ -100,7 +102,10 @@ private void EnqueueMessage(Message message) break; } if (!is_single || message_queue.All(p => p.Command != message.Command)) + { message_queue.Enqueue(message); + lastSent = TimeProvider.Current.UtcNow; + } CheckMessageQueue(); } @@ -124,7 +129,7 @@ protected override void OnReceive(object message) switch (message) { case Timer _: - RefreshPendingKnownHashes(); + OnTimer(); break; case Message msg: if (msg.Payload is PingPayload payload) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index e20cc36e78..07df4e22e1 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -297,8 +297,6 @@ private void RequestTasks(TaskSession session) { session.RemoteNode.Tell(Message.Create(MessageCommand.Mempool)); } - - session.RemoteNode.Tell(Message.Create(MessageCommand.Ping, PingPayload.Create(currentHeight))); } } } From 4847752df3f6917875eb1f0bc0b40e66aa0f9f3a Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 18:43:52 +0800 Subject: [PATCH 56/91] Remove TaskSession.StartHeight --- src/neo/Network/P2P/TaskSession.cs | 4 +--- tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs | 2 -- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index c4ca9c6004..5766b35093 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -15,7 +15,6 @@ internal class TaskSession public readonly HashSet AvailableTasks = new HashSet(); public bool HasTask => Tasks.Count > 0; - public uint StartHeight { get; } public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } @@ -25,8 +24,7 @@ public TaskSession(IActorRef node, VersionPayload version) this.IsFullNode = fullNode != null; this.RemoteNode = node; this.Version = version; - this.StartHeight = fullNode?.StartHeight ?? 0; - this.LastBlockIndex = this.StartHeight; + this.LastBlockIndex = fullNode?.StartHeight ?? 0; } } } diff --git a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs index b181eaf352..f89aa27231 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs @@ -16,7 +16,6 @@ public void CreateTest() Assert.IsNull(ses.RemoteNode); Assert.IsFalse(ses.HasTask); - Assert.AreEqual((uint)123, ses.StartHeight); Assert.AreEqual((uint)123, ses.LastBlockIndex); Assert.IsTrue(ses.IsFullNode); @@ -24,7 +23,6 @@ public void CreateTest() Assert.IsNull(ses.RemoteNode); Assert.IsFalse(ses.HasTask); - Assert.AreEqual((uint)0, ses.StartHeight); Assert.AreEqual((uint)0, ses.LastBlockIndex); Assert.IsFalse(ses.IsFullNode); } From d237a9a2fb552dfcff2bf8fa909be6afc71011ed Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 18:54:42 +0800 Subject: [PATCH 57/91] Remove TaskSession.RemoteNode --- src/neo/Network/P2P/TaskManager.cs | 30 +++++++++---------- src/neo/Network/P2P/TaskSession.cs | 5 +--- .../Network/P2P/UT_TaskSession.cs | 7 ++--- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 07df4e22e1..cc9166e2c8 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -52,7 +52,7 @@ private void OnBlock(Block _) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; - RequestTasks(session); + RequestTasks(Sender, session); } private void OnHeaders(Header[] _) @@ -61,7 +61,7 @@ private void OnHeaders(Header[] _) return; session.Tasks.Remove(HeaderTaskHash); DecrementGlobalTask(HeaderTaskHash); - RequestTasks(session); + RequestTasks(Sender, session); } private void OnNewTasks(InvPayload payload) @@ -73,7 +73,7 @@ private void OnNewTasks(InvPayload payload) uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; if (payload.Type == InventoryType.TX && currentHeight < headerHeight) { - RequestTasks(session); + RequestTasks(Sender, session); return; } HashSet hashes = new HashSet(payload.Hashes); @@ -87,7 +87,7 @@ private void OnNewTasks(InvPayload payload) hashes.Remove(globalTasks); if (hashes.Count == 0) { - RequestTasks(session); + RequestTasks(Sender, session); return; } @@ -139,11 +139,11 @@ protected override void OnReceive(object message) private void OnRegister(VersionPayload version) { Context.Watch(Sender); - TaskSession session = new TaskSession(Sender, version); + TaskSession session = new TaskSession(version); if (session.IsFullNode) - session.AvailableTasks.Add(TaskManager.MemPoolTaskHash); + session.AvailableTasks.Add(MemPoolTaskHash); sessions.Add(Sender, session); - RequestTasks(session); + RequestTasks(Sender, session); } private void OnUpdate(Update update) @@ -171,7 +171,7 @@ private void OnTaskCompleted(UInt256 hash) if (sessions.TryGetValue(Sender, out TaskSession session)) { session.Tasks.Remove(hash); - RequestTasks(session); + RequestTasks(Sender, session); } } @@ -220,8 +220,8 @@ private void OnTimer() if (session.Tasks.Remove(task.Key)) DecrementGlobalTask(task.Key); } - foreach (TaskSession session in sessions.Values) - RequestTasks(session); + foreach (var (actor, session) in sessions) + RequestTasks(actor, session); } protected override void PostStop() @@ -235,7 +235,7 @@ public static Props Props(NeoSystem system) return Akka.Actor.Props.Create(() => new TaskManager(system)).WithMailbox("task-manager-mailbox"); } - private void RequestTasks(TaskSession session) + private void RequestTasks(IActorRef remoteNode, TaskSession session) { if (session.HasTask) return; @@ -260,7 +260,7 @@ private void RequestTasks(TaskSession session) foreach (UInt256 hash in hashes) session.Tasks[hash] = DateTime.UtcNow; foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray())) - session.RemoteNode.Tell(Message.Create(MessageCommand.GetData, group)); + remoteNode.Tell(Message.Create(MessageCommand.GetData, group)); return; } } @@ -276,7 +276,7 @@ private void RequestTasks(TaskSession session) { session.Tasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); - session.RemoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlockByIndexPayload.Create(headerHeight))); + remoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlockByIndexPayload.Create(headerHeight))); } else if (currentHeight < session.LastBlockIndex) { @@ -288,14 +288,14 @@ private void RequestTasks(TaskSession session) if (!globalTasks.ContainsKey(nextHash)) break; startHeight = i; } - session.RemoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); + remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); } else if (headerHeight >= session.LastBlockIndex && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(snapshot, headerHash)?.Timestamp) { if (session.AvailableTasks.Remove(MemPoolTaskHash)) { - session.RemoteNode.Tell(Message.Create(MessageCommand.Mempool)); + remoteNode.Tell(Message.Create(MessageCommand.Mempool)); } } } diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index 5766b35093..01b4c6dd36 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -1,4 +1,3 @@ -using Akka.Actor; using Neo.Network.P2P.Capabilities; using Neo.Network.P2P.Payloads; using System; @@ -9,7 +8,6 @@ namespace Neo.Network.P2P { internal class TaskSession { - public readonly IActorRef RemoteNode; public readonly VersionPayload Version; public readonly Dictionary Tasks = new Dictionary(); public readonly HashSet AvailableTasks = new HashSet(); @@ -18,11 +16,10 @@ internal class TaskSession public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } - public TaskSession(IActorRef node, VersionPayload version) + public TaskSession(VersionPayload version) { var fullNode = version.Capabilities.OfType().FirstOrDefault(); this.IsFullNode = fullNode != null; - this.RemoteNode = node; this.Version = version; this.LastBlockIndex = fullNode?.StartHeight ?? 0; } diff --git a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs index f89aa27231..61b6aa566a 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs @@ -2,6 +2,7 @@ using Neo.Network.P2P; using Neo.Network.P2P.Capabilities; using Neo.Network.P2P.Payloads; +using System; using Xunit.Sdk; namespace Neo.UnitTests.Network.P2P @@ -12,16 +13,14 @@ public class UT_TaskSession [TestMethod] public void CreateTest() { - var ses = new TaskSession(null, new VersionPayload() { Capabilities = new NodeCapability[] { new FullNodeCapability(123) } }); + var ses = new TaskSession(new VersionPayload() { Capabilities = new NodeCapability[] { new FullNodeCapability(123) } }); - Assert.IsNull(ses.RemoteNode); Assert.IsFalse(ses.HasTask); Assert.AreEqual((uint)123, ses.LastBlockIndex); Assert.IsTrue(ses.IsFullNode); - ses = new TaskSession(null, new VersionPayload() { Capabilities = new NodeCapability[0] }); + ses = new TaskSession(new VersionPayload() { Capabilities = Array.Empty() }); - Assert.IsNull(ses.RemoteNode); Assert.IsFalse(ses.HasTask); Assert.AreEqual((uint)0, ses.LastBlockIndex); Assert.IsFalse(ses.IsFullNode); From 8b856681e5f6fecd5c2d16661c93eaad3d7c3206 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 18:57:51 +0800 Subject: [PATCH 58/91] Remove TaskSession.Version --- src/neo/Network/P2P/TaskSession.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index 01b4c6dd36..b894ae18e4 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -8,7 +8,6 @@ namespace Neo.Network.P2P { internal class TaskSession { - public readonly VersionPayload Version; public readonly Dictionary Tasks = new Dictionary(); public readonly HashSet AvailableTasks = new HashSet(); @@ -20,7 +19,6 @@ public TaskSession(VersionPayload version) { var fullNode = version.Capabilities.OfType().FirstOrDefault(); this.IsFullNode = fullNode != null; - this.Version = version; this.LastBlockIndex = fullNode?.StartHeight ?? 0; } } From 4099cf4388bb88a7fd226772b722755155e3b909 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 19:03:25 +0800 Subject: [PATCH 59/91] Update TaskSession.cs --- src/neo/Network/P2P/TaskSession.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index b894ae18e4..09609753a0 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -8,9 +8,8 @@ namespace Neo.Network.P2P { internal class TaskSession { - public readonly Dictionary Tasks = new Dictionary(); - public readonly HashSet AvailableTasks = new HashSet(); - + public Dictionary Tasks { get; } = new Dictionary(); + public HashSet AvailableTasks { get; } = new HashSet(); public bool HasTask => Tasks.Count > 0; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } From a66d1d73051c4d020c3e10d6de98b473b2139f3a Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 19:14:31 +0800 Subject: [PATCH 60/91] Remove MemPoolTaskHash --- src/neo/Network/P2P/TaskManager.cs | 14 +++----------- src/neo/Network/P2P/TaskSession.cs | 1 + 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index cc9166e2c8..09010c6720 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -24,11 +24,9 @@ private class Timer { } private static readonly TimeSpan TimerInterval = TimeSpan.FromSeconds(30); private static readonly TimeSpan TaskTimeout = TimeSpan.FromMinutes(1); - private static readonly UInt256 MemPoolTaskHash = UInt256.Parse("0x0000000000000000000000000000000000000000000000000000000000000001"); private static readonly UInt256 HeaderTaskHash = UInt256.Zero; private const int MaxConncurrentTasks = 3; - private const int PingCoolingOffPeriod = 60_000; // in ms. private readonly NeoSystem system; /// @@ -140,8 +138,6 @@ private void OnRegister(VersionPayload version) { Context.Watch(Sender); TaskSession session = new TaskSession(version); - if (session.IsFullNode) - session.AvailableTasks.Add(MemPoolTaskHash); sessions.Add(Sender, session); RequestTasks(Sender, session); } @@ -248,7 +244,6 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) // Search any similar hash that is on Singleton's knowledge, which means, on the way or already processed session.AvailableTasks.RemoveWhere(p => NativeContract.Ledger.ContainsBlock(snapshot, p)); HashSet hashes = new HashSet(session.AvailableTasks); - hashes.Remove(MemPoolTaskHash); if (hashes.Count > 0) { foreach (UInt256 hash in hashes.ToArray()) @@ -290,13 +285,10 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) } remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); } - else if (headerHeight >= session.LastBlockIndex - && TimeProvider.Current.UtcNow.ToTimestampMS() - PingCoolingOffPeriod >= NativeContract.Ledger.GetBlock(snapshot, headerHash)?.Timestamp) + else if (!session.MempoolSent) { - if (session.AvailableTasks.Remove(MemPoolTaskHash)) - { - remoteNode.Tell(Message.Create(MessageCommand.Mempool)); - } + session.MempoolSent = true; + remoteNode.Tell(Message.Create(MessageCommand.Mempool)); } } } diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index 09609753a0..a64d40f0f4 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -13,6 +13,7 @@ internal class TaskSession public bool HasTask => Tasks.Count > 0; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } + public bool MempoolSent { get; set; } public TaskSession(VersionPayload version) { From f72105aa232a52bfed71d75e0f0f0597488d0d4c Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Sat, 30 Jan 2021 19:35:37 +0800 Subject: [PATCH 61/91] Optimize --- src/neo/Ledger/HeaderCache.cs | 11 ++++++++--- src/neo/Network/P2P/TaskManager.cs | 14 +++++--------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index 5a39336e35..757f1b0a13 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -1,11 +1,13 @@ using Neo.IO.Caching; using Neo.Network.P2P.Payloads; using System; +using System.Collections; +using System.Collections.Generic; using System.Threading; namespace Neo.Ledger { - public sealed class HeaderCache : IDisposable + public sealed class HeaderCache : IDisposable, IEnumerable
{ private readonly IndexedQueue
headers = new IndexedQueue
(); private readonly ReaderWriterLockSlim readerWriterLock = new ReaderWriterLockSlim(); @@ -81,17 +83,20 @@ internal void TryRemoveFirst() } } - public Header[] GetSnapshot() + public IEnumerator
GetEnumerator() { readerWriterLock.EnterReadLock(); try { - return headers.ToArray(); + foreach (Header header in headers) + yield return header; } finally { readerWriterLock.ExitReadLock(); } } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); } } diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 09010c6720..cc793e828d 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -260,11 +260,8 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) } } - Header[] headers = Blockchain.Singleton.HeaderCache.GetSnapshot(); - Header last = headers.Length > 0 ? headers[^1] : null; uint currentHeight = NativeContract.Ledger.CurrentIndex(snapshot); - uint headerHeight = last?.Index ?? currentHeight; - UInt256 headerHash = last?.Hash ?? NativeContract.Ledger.CurrentHash(snapshot); + uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) @@ -276,12 +273,11 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) else if (currentHeight < session.LastBlockIndex) { uint startHeight = currentHeight; - for (uint i = 0; i < headers.Length; i++) + foreach (Header header in Blockchain.Singleton.HeaderCache) { - if (headers[i].Index <= currentHeight) continue; - UInt256 nextHash = headers[i].Hash; - if (!globalTasks.ContainsKey(nextHash)) break; - startHeight = i; + if (header.Index <= currentHeight) continue; + if (!globalTasks.ContainsKey(header.Hash)) break; + ++startHeight; } remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); } From 9c90f4b25139043721c22a50d6ae0b0feb80d0fb Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Sun, 31 Jan 2021 00:26:04 +0800 Subject: [PATCH 62/91] restore indextasks --- src/neo/Network/P2P/TaskManager.cs | 16 ++++++++++++---- src/neo/Network/P2P/TaskSession.cs | 1 + .../neo.UnitTests/Network/P2P/UT_TaskSession.cs | 2 ++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index cc793e828d..85057014a2 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -46,11 +46,15 @@ public TaskManager(NeoSystem system) Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } - private void OnBlock(Block _) + private void OnBlock(Block block) { - if (!sessions.TryGetValue(Sender, out TaskSession session)) + foreach (var session in sessions.Values) + { + session.IndexTasks.Remove(block.Index); + } + if (!sessions.TryGetValue(Sender, out TaskSession senderSession)) return; - RequestTasks(Sender, session); + RequestTasks(Sender, senderSession); } private void OnHeaders(Header[] _) @@ -279,7 +283,11 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) if (!globalTasks.ContainsKey(header.Hash)) break; ++startHeight; } - remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); + if (!sessions.Values.Any(p => p.IndexTasks.ContainsKey(startHeight) && p.IndexTasks[startHeight].AddSeconds(5) > TimeProvider.Current.UtcNow)) + { + session.IndexTasks[startHeight] = TimeProvider.Current.UtcNow; + remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); + } } else if (!session.MempoolSent) { diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index a64d40f0f4..b71a863e89 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -8,6 +8,7 @@ namespace Neo.Network.P2P { internal class TaskSession { + public readonly Dictionary IndexTasks = new Dictionary(); public Dictionary Tasks { get; } = new Dictionary(); public HashSet AvailableTasks { get; } = new HashSet(); public bool HasTask => Tasks.Count > 0; diff --git a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs index 61b6aa566a..7ebc57fc07 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs @@ -17,12 +17,14 @@ public void CreateTest() Assert.IsFalse(ses.HasTask); Assert.AreEqual((uint)123, ses.LastBlockIndex); + Assert.AreEqual(0, ses.IndexTasks.Count); Assert.IsTrue(ses.IsFullNode); ses = new TaskSession(new VersionPayload() { Capabilities = Array.Empty() }); Assert.IsFalse(ses.HasTask); Assert.AreEqual((uint)0, ses.LastBlockIndex); + Assert.AreEqual(0, ses.IndexTasks.Count); Assert.IsFalse(ses.IsFullNode); } } From fa5e04cef77deb3a23c9b7dad9feb8feab76e285 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 31 Jan 2021 11:34:19 +0100 Subject: [PATCH 63/91] Remove hardcoded time --- src/neo/Ledger/Blockchain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 5c872a4f2f..a4f826da47 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -266,7 +266,7 @@ private VerifyResult OnNewBlock(Block block) Persist(blockToPersist); // 15000 is the default among of seconds per block, while MilliSecondsPerBlock is the current - uint extraBlocks = (15000 - MillisecondsPerBlock) / 1000; + uint extraBlocks = (ProtocolSettings.Default.MillisecondsPerBlock - MillisecondsPerBlock) / 1000; if (blocksPersisted++ < blocksToPersistList.Count - (2 + Math.Max(0, extraBlocks))) continue; // Empirically calibrated for relaying the most recent 2 blocks persisted with 15s network From 66bb0a2cbfac9a4297da8780a1ee71027daad6bf Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 31 Jan 2021 11:35:30 +0100 Subject: [PATCH 64/91] Optimize --- src/neo/Ledger/Blockchain.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index a4f826da47..d08d9f63d1 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -260,14 +260,13 @@ private VerifyResult OnNewBlock(Block block) } int blocksPersisted = 0; + // 15000 is the default among of seconds per block, while MilliSecondsPerBlock is the current + uint extraBlocks = (ProtocolSettings.Default.MillisecondsPerBlock - MillisecondsPerBlock) / 1000; foreach (Block blockToPersist in blocksToPersistList) { block_cache_unverified.Remove(blockToPersist.Index); Persist(blockToPersist); - // 15000 is the default among of seconds per block, while MilliSecondsPerBlock is the current - uint extraBlocks = (ProtocolSettings.Default.MillisecondsPerBlock - MillisecondsPerBlock) / 1000; - if (blocksPersisted++ < blocksToPersistList.Count - (2 + Math.Max(0, extraBlocks))) continue; // Empirically calibrated for relaying the most recent 2 blocks persisted with 15s network // Increase in the rate of 1 block per second in configurations with faster blocks From e720a7247cd44be44f0db0b487dea5b1f5d26d94 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 31 Jan 2021 11:36:26 +0100 Subject: [PATCH 65/91] Revert change --- src/neo/Ledger/Blockchain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index d08d9f63d1..9e56af856c 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -261,7 +261,7 @@ private VerifyResult OnNewBlock(Block block) int blocksPersisted = 0; // 15000 is the default among of seconds per block, while MilliSecondsPerBlock is the current - uint extraBlocks = (ProtocolSettings.Default.MillisecondsPerBlock - MillisecondsPerBlock) / 1000; + uint extraBlocks = (15000 - MillisecondsPerBlock) / 1000; foreach (Block blockToPersist in blocksToPersistList) { block_cache_unverified.Remove(blockToPersist.Index); From 4cffe13209579977c402ade410d043e7f8e6665c Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Mon, 1 Feb 2021 14:57:37 +0800 Subject: [PATCH 66/91] Rename --- src/neo/Network/P2P/TaskManager.cs | 16 ++++++++-------- src/neo/Network/P2P/TaskSession.cs | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 85057014a2..eef0c55e9b 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -61,7 +61,7 @@ private void OnHeaders(Header[] _) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; - session.Tasks.Remove(HeaderTaskHash); + session.InvTasks.Remove(HeaderTaskHash); DecrementGlobalTask(HeaderTaskHash); RequestTasks(Sender, session); } @@ -97,7 +97,7 @@ private void OnNewTasks(InvPayload payload) foreach (UInt256 hash in hashes) { IncrementGlobalTask(hash); - session.Tasks[hash] = DateTime.UtcNow; + session.InvTasks[hash] = DateTime.UtcNow; } foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, hashes.ToArray())) @@ -170,7 +170,7 @@ private void OnTaskCompleted(UInt256 hash) ms.AvailableTasks.Remove(hash); if (sessions.TryGetValue(Sender, out TaskSession session)) { - session.Tasks.Remove(hash); + session.InvTasks.Remove(hash); RequestTasks(Sender, session); } } @@ -206,7 +206,7 @@ private void OnTerminated(IActorRef actor) { if (!sessions.TryGetValue(actor, out TaskSession session)) return; - foreach (UInt256 hash in session.Tasks.Keys) + foreach (UInt256 hash in session.InvTasks.Keys) DecrementGlobalTask(hash); sessions.Remove(actor); } @@ -214,10 +214,10 @@ private void OnTerminated(IActorRef actor) private void OnTimer() { foreach (TaskSession session in sessions.Values) - foreach (var task in session.Tasks.ToArray()) + foreach (var task in session.InvTasks.ToArray()) if (TimeProvider.Current.UtcNow - task.Value > TaskTimeout) { - if (session.Tasks.Remove(task.Key)) + if (session.InvTasks.Remove(task.Key)) DecrementGlobalTask(task.Key); } foreach (var (actor, session) in sessions) @@ -257,7 +257,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) } session.AvailableTasks.Remove(hashes); foreach (UInt256 hash in hashes) - session.Tasks[hash] = DateTime.UtcNow; + session.InvTasks[hash] = DateTime.UtcNow; foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray())) remoteNode.Tell(Message.Create(MessageCommand.GetData, group)); return; @@ -270,7 +270,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) // If not HeaderTask pending to be processed it should ask for more Blocks if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) { - session.Tasks[HeaderTaskHash] = DateTime.UtcNow; + session.InvTasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); remoteNode.Tell(Message.Create(MessageCommand.GetHeaders, GetBlockByIndexPayload.Create(headerHeight))); } diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index b71a863e89..01ea286108 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -8,10 +8,10 @@ namespace Neo.Network.P2P { internal class TaskSession { - public readonly Dictionary IndexTasks = new Dictionary(); - public Dictionary Tasks { get; } = new Dictionary(); + public Dictionary InvTasks { get; } = new Dictionary(); + public Dictionary IndexTasks { get; } = new Dictionary(); public HashSet AvailableTasks { get; } = new HashSet(); - public bool HasTask => Tasks.Count > 0; + public bool HasTask => InvTasks.Count > 0; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } public bool MempoolSent { get; set; } From 1834852a868b1985c31711f12a57c80b1d1b7af9 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Mon, 1 Feb 2021 15:07:38 +0800 Subject: [PATCH 67/91] Optimize --- src/neo/Network/P2P/TaskManager.cs | 28 ++++++++-------------------- 1 file changed, 8 insertions(+), 20 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index eef0c55e9b..ca01a7545d 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -46,17 +46,6 @@ public TaskManager(NeoSystem system) Context.System.EventStream.Subscribe(Self, typeof(Blockchain.RelayResult)); } - private void OnBlock(Block block) - { - foreach (var session in sessions.Values) - { - session.IndexTasks.Remove(block.Index); - } - if (!sessions.TryGetValue(Sender, out TaskSession senderSession)) - return; - RequestTasks(Sender, senderSession); - } - private void OnHeaders(Header[] _) { if (!sessions.TryGetValue(Sender, out TaskSession session)) @@ -120,14 +109,11 @@ protected override void OnReceive(object message) case RestartTasks restart: OnRestartTasks(restart.Payload); break; - case Block block: - OnBlock(block); - break; case Header[] headers: OnHeaders(headers); break; case IInventory inventory: - OnTaskCompleted(inventory.Hash); + OnTaskCompleted(inventory); break; case Timer _: OnTimer(); @@ -162,15 +148,17 @@ private void OnRestartTasks(InvPayload payload) system.LocalNode.Tell(Message.Create(MessageCommand.GetData, group)); } - private void OnTaskCompleted(UInt256 hash) + private void OnTaskCompleted(IInventory inventory) { - knownHashes.Add(hash); - globalTasks.Remove(hash); + knownHashes.Add(inventory.Hash); + globalTasks.Remove(inventory.Hash); foreach (TaskSession ms in sessions.Values) - ms.AvailableTasks.Remove(hash); + ms.AvailableTasks.Remove(inventory.Hash); if (sessions.TryGetValue(Sender, out TaskSession session)) { - session.InvTasks.Remove(hash); + session.InvTasks.Remove(inventory.Hash); + if (inventory is Block block) + session.IndexTasks.Remove(block.Index); RequestTasks(Sender, session); } } From 73edcfd1dfc2466a47048eabcaf720fbfd9e7445 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Mon, 1 Feb 2021 16:50:53 +0800 Subject: [PATCH 68/91] reset TimeProvider.Current.UtcNow --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index ca01a7545d..98942d907f 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -86,7 +86,7 @@ private void OnNewTasks(InvPayload payload) foreach (UInt256 hash in hashes) { IncrementGlobalTask(hash); - session.InvTasks[hash] = DateTime.UtcNow; + session.InvTasks[hash] = TimeProvider.Current.UtcNow; } foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, hashes.ToArray())) From a8713a3a49aa36c30808189bc05468076a9c72a4 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Mon, 1 Feb 2021 17:02:55 +0800 Subject: [PATCH 69/91] use count --- src/neo/Network/P2P/TaskManager.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 98942d907f..c8a6568478 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -273,8 +273,10 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) } if (!sessions.Values.Any(p => p.IndexTasks.ContainsKey(startHeight) && p.IndexTasks[startHeight].AddSeconds(5) > TimeProvider.Current.UtcNow)) { - session.IndexTasks[startHeight] = TimeProvider.Current.UtcNow; - remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight))); + for (uint i = startHeight; i < session.LastBlockIndex; i++) + session.IndexTasks[i] = TimeProvider.Current.UtcNow; + short count = (short)(session.LastBlockIndex - startHeight); + remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, count))); } } else if (!session.MempoolSent) From 2b366e49674d25881ba0babe89d3d97b5d93c489 Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Mon, 1 Feb 2021 17:12:34 +0800 Subject: [PATCH 70/91] fix --- src/neo/Network/P2P/TaskManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c8a6568478..703a2e7563 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -273,9 +273,9 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) } if (!sessions.Values.Any(p => p.IndexTasks.ContainsKey(startHeight) && p.IndexTasks[startHeight].AddSeconds(5) > TimeProvider.Current.UtcNow)) { - for (uint i = startHeight; i < session.LastBlockIndex; i++) + for (uint i = startHeight; i <= session.LastBlockIndex; i++) session.IndexTasks[i] = TimeProvider.Current.UtcNow; - short count = (short)(session.LastBlockIndex - startHeight); + short count = (short)(session.LastBlockIndex - startHeight + 1); remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, count))); } } From 439699f67860872b6cffac265a1b7a374553c7ba Mon Sep 17 00:00:00 2001 From: Tommo-L Date: Mon, 1 Feb 2021 17:15:36 +0800 Subject: [PATCH 71/91] fix (2) --- src/neo/Network/P2P/TaskManager.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 703a2e7563..1faf9ede7e 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -273,9 +273,9 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) } if (!sessions.Values.Any(p => p.IndexTasks.ContainsKey(startHeight) && p.IndexTasks[startHeight].AddSeconds(5) > TimeProvider.Current.UtcNow)) { - for (uint i = startHeight; i <= session.LastBlockIndex; i++) + short count = 0; + for (uint i = startHeight; i <= session.LastBlockIndex && count < HeadersPayload.MaxHeadersCount; i++, count++) session.IndexTasks[i] = TimeProvider.Current.UtcNow; - short count = (short)(session.LastBlockIndex - startHeight + 1); remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, count))); } } From eda2db7db2423ada44cd5fde14424751267e565d Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Mon, 1 Feb 2021 22:03:55 +0800 Subject: [PATCH 72/91] Add globalIndexTasks --- src/neo/Network/P2P/TaskManager.cs | 101 ++++++++++++++++++++--------- 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 1faf9ede7e..bd17406a10 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -33,10 +33,12 @@ private class Timer { } /// A set of known hashes, of inventories or payloads, already received. ///
private readonly HashSetCache knownHashes; - private readonly Dictionary globalTasks = new Dictionary(); + private readonly Dictionary globalInvTasks = new Dictionary(); + private readonly Dictionary globalIndexTasks = new Dictionary(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); - private bool HasHeaderTask => globalTasks.ContainsKey(HeaderTaskHash); + + private bool HasHeaderTask => globalInvTasks.ContainsKey(HeaderTaskHash); public TaskManager(NeoSystem system) { @@ -50,8 +52,8 @@ private void OnHeaders(Header[] _) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; - session.InvTasks.Remove(HeaderTaskHash); - DecrementGlobalTask(HeaderTaskHash); + if (session.InvTasks.Remove(HeaderTaskHash)) + DecrementGlobalTask(HeaderTaskHash); RequestTasks(Sender, session); } @@ -72,10 +74,10 @@ private void OnNewTasks(InvPayload payload) hashes.Remove(knownHashes); // Add to AvailableTasks the ones, of type InventoryType.Block, that are global (already under process by other sessions) if (payload.Type == InventoryType.Block) - session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p))); + session.AvailableTasks.UnionWith(hashes.Where(p => globalInvTasks.ContainsKey(p))); // Remove those that are already in process by other sessions - hashes.Remove(globalTasks); + hashes.Remove(globalInvTasks); if (hashes.Count == 0) { RequestTasks(Sender, session); @@ -143,21 +145,24 @@ private void OnRestartTasks(InvPayload payload) { knownHashes.ExceptWith(payload.Hashes); foreach (UInt256 hash in payload.Hashes) - globalTasks.Remove(hash); + globalInvTasks.Remove(hash); foreach (InvPayload group in InvPayload.CreateGroup(payload.Type, payload.Hashes)) system.LocalNode.Tell(Message.Create(MessageCommand.GetData, group)); } private void OnTaskCompleted(IInventory inventory) { + Block block = inventory as Block; knownHashes.Add(inventory.Hash); - globalTasks.Remove(inventory.Hash); + globalInvTasks.Remove(inventory.Hash); + if (block is not null) + globalIndexTasks.Remove(block.Index); foreach (TaskSession ms in sessions.Values) ms.AvailableTasks.Remove(inventory.Hash); if (sessions.TryGetValue(Sender, out TaskSession session)) { session.InvTasks.Remove(inventory.Hash); - if (inventory is Block block) + if (block is not null) session.IndexTasks.Remove(block.Index); RequestTasks(Sender, session); } @@ -166,27 +171,54 @@ private void OnTaskCompleted(IInventory inventory) [MethodImpl(MethodImplOptions.AggressiveInlining)] private void DecrementGlobalTask(UInt256 hash) { - if (globalTasks.TryGetValue(hash, out var value)) + if (globalInvTasks.TryGetValue(hash, out var value)) { if (value == 1) - globalTasks.Remove(hash); + globalInvTasks.Remove(hash); else - globalTasks[hash] = value - 1; + globalInvTasks[hash] = value - 1; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void DecrementGlobalTask(uint index) + { + if (globalIndexTasks.TryGetValue(index, out var value)) + { + if (value == 1) + globalIndexTasks.Remove(index); + else + globalIndexTasks[index] = value - 1; } } [MethodImpl(MethodImplOptions.AggressiveInlining)] private bool IncrementGlobalTask(UInt256 hash) { - if (!globalTasks.TryGetValue(hash, out var value)) + if (!globalInvTasks.TryGetValue(hash, out var value)) + { + globalInvTasks[hash] = 1; + return true; + } + if (value >= MaxConncurrentTasks) + return false; + + globalInvTasks[hash] = value + 1; + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool IncrementGlobalTask(uint index) + { + if (!globalIndexTasks.TryGetValue(index, out var value)) { - globalTasks[hash] = 1; + globalIndexTasks[index] = 1; return true; } if (value >= MaxConncurrentTasks) return false; - globalTasks[hash] = value + 1; + globalIndexTasks[index] = value + 1; return true; } @@ -196,18 +228,28 @@ private void OnTerminated(IActorRef actor) return; foreach (UInt256 hash in session.InvTasks.Keys) DecrementGlobalTask(hash); + foreach (uint index in session.IndexTasks.Keys) + DecrementGlobalTask(index); sessions.Remove(actor); } private void OnTimer() { foreach (TaskSession session in sessions.Values) - foreach (var task in session.InvTasks.ToArray()) - if (TimeProvider.Current.UtcNow - task.Value > TaskTimeout) + { + foreach (var (hash, time) in session.InvTasks.ToArray()) + if (TimeProvider.Current.UtcNow - time > TaskTimeout) + { + if (session.InvTasks.Remove(hash)) + DecrementGlobalTask(hash); + } + foreach (var (index, time) in session.IndexTasks.ToArray()) + if (TimeProvider.Current.UtcNow - time > TaskTimeout) { - if (session.InvTasks.Remove(task.Key)) - DecrementGlobalTask(task.Key); + if (session.IndexTasks.Remove(index)) + DecrementGlobalTask(index); } + } foreach (var (actor, session) in sessions) RequestTasks(actor, session); } @@ -256,7 +298,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; // When the number of AvailableTasks is no more than 0, no pending tasks of InventoryType.Block, it should process pending the tasks of headers // If not HeaderTask pending to be processed it should ask for more Blocks - if ((!HasHeaderTask || globalTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) + if ((!HasHeaderTask || globalInvTasks[HeaderTaskHash] < MaxConncurrentTasks) && headerHeight < session.LastBlockIndex && !Blockchain.Singleton.HeaderCache.Full) { session.InvTasks[HeaderTaskHash] = DateTime.UtcNow; IncrementGlobalTask(HeaderTaskHash); @@ -265,19 +307,16 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) else if (currentHeight < session.LastBlockIndex) { uint startHeight = currentHeight; - foreach (Header header in Blockchain.Singleton.HeaderCache) - { - if (header.Index <= currentHeight) continue; - if (!globalTasks.ContainsKey(header.Hash)) break; - ++startHeight; - } - if (!sessions.Values.Any(p => p.IndexTasks.ContainsKey(startHeight) && p.IndexTasks[startHeight].AddSeconds(5) > TimeProvider.Current.UtcNow)) + while (globalIndexTasks.ContainsKey(++startHeight)) { } + uint endHeight = startHeight; + while (!globalIndexTasks.ContainsKey(++endHeight) && endHeight <= session.LastBlockIndex) { } + uint count = Math.Min(endHeight - startHeight, HeadersPayload.MaxHeadersCount); + for (uint i = 0; i < count; i++) { - short count = 0; - for (uint i = startHeight; i <= session.LastBlockIndex && count < HeadersPayload.MaxHeadersCount; i++, count++) - session.IndexTasks[i] = TimeProvider.Current.UtcNow; - remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, count))); + session.IndexTasks[startHeight + i] = TimeProvider.Current.UtcNow; + IncrementGlobalTask(startHeight + i); } + remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, (short)count))); } else if (!session.MempoolSent) { From 620a95b8325ea46f1fb8367a182e4a3c7f980dae Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 00:25:46 +0800 Subject: [PATCH 73/91] fix --- src/neo/Network/P2P/TaskSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index 01ea286108..55d88c824a 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -11,7 +11,7 @@ internal class TaskSession public Dictionary InvTasks { get; } = new Dictionary(); public Dictionary IndexTasks { get; } = new Dictionary(); public HashSet AvailableTasks { get; } = new HashSet(); - public bool HasTask => InvTasks.Count > 0; + public bool HasTask => InvTasks.Count > 0 || IndexTasks.Count > 0; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } public bool MempoolSent { get; set; } From 1669be66efc254d9e7890a992ca9a80a3260e1a7 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 00:51:13 +0800 Subject: [PATCH 74/91] fix --- src/neo/Network/P2P/TaskManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index bd17406a10..60ea1c8d9f 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -308,6 +308,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) { uint startHeight = currentHeight; while (globalIndexTasks.ContainsKey(++startHeight)) { } + if (startHeight > session.LastBlockIndex) return; uint endHeight = startHeight; while (!globalIndexTasks.ContainsKey(++endHeight) && endHeight <= session.LastBlockIndex) { } uint count = Math.Min(endHeight - startHeight, HeadersPayload.MaxHeadersCount); From 804d30b43354214a2a9e55d81093dd1c41662e57 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 01:15:57 +0800 Subject: [PATCH 75/91] Fix --- src/neo/Ledger/Blockchain.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index 9e56af856c..cf7425e746 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -371,7 +371,6 @@ private void Persist(Block block) { using (SnapshotCache snapshot = GetSnapshot()) { - HeaderCache.TryRemoveFirst(); List all_application_executed = new List(); using (ApplicationEngine engine = ApplicationEngine.Create(TriggerType.OnPersist, null, snapshot, block)) { @@ -436,6 +435,7 @@ private void Persist(Block block) } block_cache.TryRemove(block.PrevHash, out _); Context.System.EventStream.Publish(new PersistCompleted { Block = block }); + HeaderCache.TryRemoveFirst(); } public static Props Props(NeoSystem system, IStore store) From 12b2f6dda3503a5f7a3fc713f52b474d1288b322 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 01:18:26 +0800 Subject: [PATCH 76/91] Assert --- src/neo/Ledger/Blockchain.cs | 4 +++- src/neo/Ledger/HeaderCache.cs | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/neo/Ledger/Blockchain.cs b/src/neo/Ledger/Blockchain.cs index cf7425e746..8a923e87d0 100644 --- a/src/neo/Ledger/Blockchain.cs +++ b/src/neo/Ledger/Blockchain.cs @@ -16,6 +16,7 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; +using System.Diagnostics; using System.Linq; using System.Threading; @@ -435,7 +436,8 @@ private void Persist(Block block) } block_cache.TryRemove(block.PrevHash, out _); Context.System.EventStream.Publish(new PersistCompleted { Block = block }); - HeaderCache.TryRemoveFirst(); + if (HeaderCache.TryRemoveFirst(out Header header)) + Debug.Assert(header.Index == block.Index); } public static Props Props(NeoSystem system, IStore store) diff --git a/src/neo/Ledger/HeaderCache.cs b/src/neo/Ledger/HeaderCache.cs index 757f1b0a13..ea3380baa9 100644 --- a/src/neo/Ledger/HeaderCache.cs +++ b/src/neo/Ledger/HeaderCache.cs @@ -70,12 +70,12 @@ internal void Add(Header header) } } - internal void TryRemoveFirst() + internal bool TryRemoveFirst(out Header header) { readerWriterLock.EnterWriteLock(); try { - headers.TryDequeue(out _); + return headers.TryDequeue(out header); } finally { From d14eb19432e4617323cb56058f2f1ba57ee78236 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 12:32:17 +0800 Subject: [PATCH 77/91] restore logic --- src/neo/Network/P2P/TaskManager.cs | 69 ++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 60ea1c8d9f..683c8e4253 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -1,5 +1,6 @@ using Akka.Actor; using Akka.Configuration; +using Akka.IO; using Neo.IO.Actors; using Neo.IO.Caching; using Neo.Ledger; @@ -35,6 +36,7 @@ private class Timer { } private readonly HashSetCache knownHashes; private readonly Dictionary globalInvTasks = new Dictionary(); private readonly Dictionary globalIndexTasks = new Dictionary(); + private readonly Dictionary> receivedBlockIndex = new Dictionary>(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); @@ -57,6 +59,30 @@ private void OnHeaders(Header[] _) RequestTasks(Sender, session); } + private void OnInvalidBlock(Block invalidBlock) + { + receivedBlockIndex.TryGetValue(invalidBlock.Index, out var record); + if (record is null) return; + var invalidSenders = record.Where(p => p.Value == invalidBlock.Hash).Select(p => p.Key); + foreach (var invalidSender in invalidSenders) + { + sessions.TryGetValue(invalidSender, out TaskSession session); + if (session != null) + { + session.IndexTasks.Remove(invalidBlock.Index); + } + Sender.Tell(Tcp.Abort.Instance); + } + receivedBlockIndex.Remove(invalidBlock.Index); + foreach (var pair in sessions) + { + if (!invalidSenders.Contains(pair.Key)) + { + RequestTasks(pair.Key, pair.Value); + } + } + } + private void OnNewTasks(InvPayload payload) { if (!sessions.TryGetValue(Sender, out TaskSession session)) @@ -95,6 +121,19 @@ private void OnNewTasks(InvPayload payload) Sender.Tell(Message.Create(MessageCommand.GetData, group)); } + private void OnPersistCompleted(Block block) + { + if (receivedBlockIndex.TryGetValue(block.Index, out var record)) + { + var validPair = record.Where(p => p.Value == block.Hash).FirstOrDefault(); + if (!default(KeyValuePair).Equals(validPair) && sessions.TryGetValue(validPair.Key, out TaskSession session)) + { + RequestTasks(validPair.Key, session); + } + receivedBlockIndex.Remove(block.Index); + } + } + protected override void OnReceive(object message) { switch (message) @@ -117,6 +156,13 @@ protected override void OnReceive(object message) case IInventory inventory: OnTaskCompleted(inventory); break; + case Blockchain.PersistCompleted pc: + OnPersistCompleted(pc.Block); + break; + case Blockchain.RelayResult rr: + if (rr.Inventory is Block invalidBlock && rr.Result == VerifyResult.Invalid) + OnInvalidBlock(invalidBlock); + break; case Timer _: OnTimer(); break; @@ -163,7 +209,30 @@ private void OnTaskCompleted(IInventory inventory) { session.InvTasks.Remove(inventory.Hash); if (block is not null) + { session.IndexTasks.Remove(block.Index); + if (receivedBlockIndex.TryGetValue(block.Index, out var record)) + { + if (record.TryGetValue(Sender, out UInt256 hash)) + { + if (hash != inventory.Hash) + { + Sender.Tell(Tcp.Abort.Instance); + return; + } + } + else + { + record.TryAdd(Sender, block.Hash); + } + } + else + { + record = new Dictionary(); + record.TryAdd(Sender, block.Hash); + receivedBlockIndex.TryAdd(block.Index, record); + } + } RequestTasks(Sender, session); } } From 3540a5469314552d899adb9d14793f1a8c760fce Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 13:45:20 +0800 Subject: [PATCH 78/91] Optimize --- src/neo/Network/P2P/TaskManager.cs | 57 ++++++++---------------------- src/neo/Network/P2P/TaskSession.cs | 1 + 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 683c8e4253..8569e71e98 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -36,7 +36,6 @@ private class Timer { } private readonly HashSetCache knownHashes; private readonly Dictionary globalInvTasks = new Dictionary(); private readonly Dictionary globalIndexTasks = new Dictionary(); - private readonly Dictionary> receivedBlockIndex = new Dictionary>(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); @@ -61,25 +60,14 @@ private void OnHeaders(Header[] _) private void OnInvalidBlock(Block invalidBlock) { - receivedBlockIndex.TryGetValue(invalidBlock.Index, out var record); - if (record is null) return; - var invalidSenders = record.Where(p => p.Value == invalidBlock.Hash).Select(p => p.Key); - foreach (var invalidSender in invalidSenders) - { - sessions.TryGetValue(invalidSender, out TaskSession session); - if (session != null) - { - session.IndexTasks.Remove(invalidBlock.Index); - } - Sender.Tell(Tcp.Abort.Instance); - } - receivedBlockIndex.Remove(invalidBlock.Index); - foreach (var pair in sessions) + foreach (var (actor, session) in sessions) { - if (!invalidSenders.Contains(pair.Key)) - { - RequestTasks(pair.Key, pair.Value); - } + if (!session.ReceivedBlock.Remove(invalidBlock.Index, out Block block)) + continue; + if (block.Hash == invalidBlock.Hash) + actor.Tell(Tcp.Abort.Instance); + else + RequestTasks(actor, session); } } @@ -123,15 +111,9 @@ private void OnNewTasks(InvPayload payload) private void OnPersistCompleted(Block block) { - if (receivedBlockIndex.TryGetValue(block.Index, out var record)) - { - var validPair = record.Where(p => p.Value == block.Hash).FirstOrDefault(); - if (!default(KeyValuePair).Equals(validPair) && sessions.TryGetValue(validPair.Key, out TaskSession session)) - { - RequestTasks(validPair.Key, session); - } - receivedBlockIndex.Remove(block.Index); - } + foreach (var (actor, session) in sessions) + if (session.ReceivedBlock.Remove(block.Index)) + RequestTasks(actor, session); } protected override void OnReceive(object message) @@ -211,26 +193,17 @@ private void OnTaskCompleted(IInventory inventory) if (block is not null) { session.IndexTasks.Remove(block.Index); - if (receivedBlockIndex.TryGetValue(block.Index, out var record)) + if (session.ReceivedBlock.TryGetValue(block.Index, out var block_old)) { - if (record.TryGetValue(Sender, out UInt256 hash)) - { - if (hash != inventory.Hash) - { - Sender.Tell(Tcp.Abort.Instance); - return; - } - } - else + if (block.Hash != block_old.Hash) { - record.TryAdd(Sender, block.Hash); + Sender.Tell(Tcp.Abort.Instance); + return; } } else { - record = new Dictionary(); - record.TryAdd(Sender, block.Hash); - receivedBlockIndex.TryAdd(block.Index, record); + session.ReceivedBlock.Add(block.Index, block); } } RequestTasks(Sender, session); diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index 55d88c824a..cd4a45ea80 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -11,6 +11,7 @@ internal class TaskSession public Dictionary InvTasks { get; } = new Dictionary(); public Dictionary IndexTasks { get; } = new Dictionary(); public HashSet AvailableTasks { get; } = new HashSet(); + public Dictionary ReceivedBlock { get; } = new Dictionary(); public bool HasTask => InvTasks.Count > 0 || IndexTasks.Count > 0; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } From d695e1b1c405a0162443266bd7cb1f7af88772c5 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 13:51:02 +0800 Subject: [PATCH 79/91] Fix --- src/neo/Network/P2P/TaskManager.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 8569e71e98..afc00b7af6 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -61,14 +61,9 @@ private void OnHeaders(Header[] _) private void OnInvalidBlock(Block invalidBlock) { foreach (var (actor, session) in sessions) - { - if (!session.ReceivedBlock.Remove(invalidBlock.Index, out Block block)) - continue; - if (block.Hash == invalidBlock.Hash) - actor.Tell(Tcp.Abort.Instance); - else - RequestTasks(actor, session); - } + if (session.ReceivedBlock.TryGetValue(invalidBlock.Index, out Block block)) + if (block.Hash == invalidBlock.Hash) + actor.Tell(Tcp.Abort.Instance); } private void OnNewTasks(InvPayload payload) From 51a879e14f22cfdd3ec9b4c9283598e402edfcc0 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Tue, 2 Feb 2021 13:55:27 +0800 Subject: [PATCH 80/91] Update TaskManager.cs --- src/neo/Network/P2P/TaskManager.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index afc00b7af6..c60ef45cc2 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -70,6 +70,7 @@ private void OnNewTasks(InvPayload payload) { if (!sessions.TryGetValue(Sender, out TaskSession session)) return; + // Do not accept payload of type InventoryType.TX if not synced on HeaderHeight uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; @@ -78,6 +79,7 @@ private void OnNewTasks(InvPayload payload) RequestTasks(Sender, session); return; } + HashSet hashes = new HashSet(payload.Hashes); // Remove all previously processed knownHashes from the list that is being requested hashes.Remove(knownHashes); From 9692809a28744427acaba806286b2a5347be188f Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 14:01:12 +0800 Subject: [PATCH 81/91] add a comparison --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c60ef45cc2..218ccc1044 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -109,7 +109,7 @@ private void OnNewTasks(InvPayload payload) private void OnPersistCompleted(Block block) { foreach (var (actor, session) in sessions) - if (session.ReceivedBlock.Remove(block.Index)) + if (session.ReceivedBlock.Remove(block.Index, out Block receivedBlock) && block.Hash == receivedBlock.Hash) RequestTasks(actor, session); } From 2354dfdef64d56ebbc36fcb7a1583ec4904df3e8 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 14:11:56 +0800 Subject: [PATCH 82/91] optimize --- src/neo/Network/P2P/TaskManager.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 218ccc1044..0fab0e446e 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -109,8 +109,13 @@ private void OnNewTasks(InvPayload payload) private void OnPersistCompleted(Block block) { foreach (var (actor, session) in sessions) - if (session.ReceivedBlock.Remove(block.Index, out Block receivedBlock) && block.Hash == receivedBlock.Hash) - RequestTasks(actor, session); + if (session.ReceivedBlock.Remove(block.Index, out Block receivedBlock)) + { + if (block.Hash == receivedBlock.Hash) + RequestTasks(actor, session); + else + actor.Tell(Tcp.Abort.Instance); + } } protected override void OnReceive(object message) From a860c4539a69127223cd8382d4bb5e1f47a45b4f Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 17:59:38 +0800 Subject: [PATCH 83/91] optimize globalindextaks --- src/neo/Network/P2P/TaskManager.cs | 41 +++++++++++++++++------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 0fab0e446e..c1590657a4 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -35,7 +35,7 @@ private class Timer { } ///
private readonly HashSetCache knownHashes; private readonly Dictionary globalInvTasks = new Dictionary(); - private readonly Dictionary globalIndexTasks = new Dictionary(); + private readonly Dictionary> globalIndexTasks = new Dictionary>(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); @@ -225,14 +225,14 @@ private void DecrementGlobalTask(UInt256 hash) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void DecrementGlobalTask(uint index) + private void DecrementGlobalTask(uint index, IActorRef actor) { if (globalIndexTasks.TryGetValue(index, out var value)) { - if (value == 1) - globalIndexTasks.Remove(index); - else - globalIndexTasks[index] = value - 1; + if (value.Remove(actor)) + { + if (value.Count == 0) globalIndexTasks.Remove(index); + } } } @@ -252,17 +252,19 @@ private bool IncrementGlobalTask(UInt256 hash) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IncrementGlobalTask(uint index) + private bool IncrementGlobalTask(uint index, IActorRef actor, uint count) { if (!globalIndexTasks.TryGetValue(index, out var value)) { - globalIndexTasks[index] = 1; + value = new Dictionary(); + value[actor] = count; + globalIndexTasks[index] = value; return true; } - if (value >= MaxConncurrentTasks) + if (value.Count >= MaxConncurrentTasks) return false; - globalIndexTasks[index] = value + 1; + globalIndexTasks[index][actor] = count; return true; } @@ -273,13 +275,13 @@ private void OnTerminated(IActorRef actor) foreach (UInt256 hash in session.InvTasks.Keys) DecrementGlobalTask(hash); foreach (uint index in session.IndexTasks.Keys) - DecrementGlobalTask(index); + DecrementGlobalTask(index, actor); sessions.Remove(actor); } private void OnTimer() { - foreach (TaskSession session in sessions.Values) + foreach (var (actor, session) in sessions) { foreach (var (hash, time) in session.InvTasks.ToArray()) if (TimeProvider.Current.UtcNow - time > TaskTimeout) @@ -291,7 +293,7 @@ private void OnTimer() if (TimeProvider.Current.UtcNow - time > TaskTimeout) { if (session.IndexTasks.Remove(index)) - DecrementGlobalTask(index); + DecrementGlobalTask(index, actor); } } foreach (var (actor, session) in sessions) @@ -311,8 +313,6 @@ public static Props Props(NeoSystem system) private void RequestTasks(IActorRef remoteNode, TaskSession session) { - if (session.HasTask) return; - DataCache snapshot = Blockchain.Singleton.View; // If there are pending tasks of InventoryType.Block we should process them @@ -351,7 +351,14 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) else if (currentHeight < session.LastBlockIndex) { uint startHeight = currentHeight; - while (globalIndexTasks.ContainsKey(++startHeight)) { } + foreach (var (index, pair) in globalIndexTasks) + { + foreach (uint _count in pair.Values) + { + uint maxRequiredHeight = index + _count; + startHeight = startHeight < maxRequiredHeight ? maxRequiredHeight : startHeight; + } + } if (startHeight > session.LastBlockIndex) return; uint endHeight = startHeight; while (!globalIndexTasks.ContainsKey(++endHeight) && endHeight <= session.LastBlockIndex) { } @@ -359,7 +366,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) for (uint i = 0; i < count; i++) { session.IndexTasks[startHeight + i] = TimeProvider.Current.UtcNow; - IncrementGlobalTask(startHeight + i); + IncrementGlobalTask(startHeight + i, remoteNode, count); } remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, (short)count))); } From a972e457ec9f92027c3627e8dc3258bafa62735d Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 19:55:19 +0800 Subject: [PATCH 84/91] Revert "optimize globalindextaks" This reverts commit a860c4539a69127223cd8382d4bb5e1f47a45b4f. --- src/neo/Network/P2P/TaskManager.cs | 41 +++++++++++++----------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c1590657a4..0fab0e446e 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -35,7 +35,7 @@ private class Timer { } /// private readonly HashSetCache knownHashes; private readonly Dictionary globalInvTasks = new Dictionary(); - private readonly Dictionary> globalIndexTasks = new Dictionary>(); + private readonly Dictionary globalIndexTasks = new Dictionary(); private readonly Dictionary sessions = new Dictionary(); private readonly ICancelable timer = Context.System.Scheduler.ScheduleTellRepeatedlyCancelable(TimerInterval, TimerInterval, Context.Self, new Timer(), ActorRefs.NoSender); @@ -225,14 +225,14 @@ private void DecrementGlobalTask(UInt256 hash) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void DecrementGlobalTask(uint index, IActorRef actor) + private void DecrementGlobalTask(uint index) { if (globalIndexTasks.TryGetValue(index, out var value)) { - if (value.Remove(actor)) - { - if (value.Count == 0) globalIndexTasks.Remove(index); - } + if (value == 1) + globalIndexTasks.Remove(index); + else + globalIndexTasks[index] = value - 1; } } @@ -252,19 +252,17 @@ private bool IncrementGlobalTask(UInt256 hash) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private bool IncrementGlobalTask(uint index, IActorRef actor, uint count) + private bool IncrementGlobalTask(uint index) { if (!globalIndexTasks.TryGetValue(index, out var value)) { - value = new Dictionary(); - value[actor] = count; - globalIndexTasks[index] = value; + globalIndexTasks[index] = 1; return true; } - if (value.Count >= MaxConncurrentTasks) + if (value >= MaxConncurrentTasks) return false; - globalIndexTasks[index][actor] = count; + globalIndexTasks[index] = value + 1; return true; } @@ -275,13 +273,13 @@ private void OnTerminated(IActorRef actor) foreach (UInt256 hash in session.InvTasks.Keys) DecrementGlobalTask(hash); foreach (uint index in session.IndexTasks.Keys) - DecrementGlobalTask(index, actor); + DecrementGlobalTask(index); sessions.Remove(actor); } private void OnTimer() { - foreach (var (actor, session) in sessions) + foreach (TaskSession session in sessions.Values) { foreach (var (hash, time) in session.InvTasks.ToArray()) if (TimeProvider.Current.UtcNow - time > TaskTimeout) @@ -293,7 +291,7 @@ private void OnTimer() if (TimeProvider.Current.UtcNow - time > TaskTimeout) { if (session.IndexTasks.Remove(index)) - DecrementGlobalTask(index, actor); + DecrementGlobalTask(index); } } foreach (var (actor, session) in sessions) @@ -313,6 +311,8 @@ public static Props Props(NeoSystem system) private void RequestTasks(IActorRef remoteNode, TaskSession session) { + if (session.HasTask) return; + DataCache snapshot = Blockchain.Singleton.View; // If there are pending tasks of InventoryType.Block we should process them @@ -351,14 +351,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) else if (currentHeight < session.LastBlockIndex) { uint startHeight = currentHeight; - foreach (var (index, pair) in globalIndexTasks) - { - foreach (uint _count in pair.Values) - { - uint maxRequiredHeight = index + _count; - startHeight = startHeight < maxRequiredHeight ? maxRequiredHeight : startHeight; - } - } + while (globalIndexTasks.ContainsKey(++startHeight)) { } if (startHeight > session.LastBlockIndex) return; uint endHeight = startHeight; while (!globalIndexTasks.ContainsKey(++endHeight) && endHeight <= session.LastBlockIndex) { } @@ -366,7 +359,7 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) for (uint i = 0; i < count; i++) { session.IndexTasks[startHeight + i] = TimeProvider.Current.UtcNow; - IncrementGlobalTask(startHeight + i, remoteNode, count); + IncrementGlobalTask(startHeight + i); } remoteNode.Tell(Message.Create(MessageCommand.GetBlockByIndex, GetBlockByIndexPayload.Create(startHeight, (short)count))); } From 565fa05685732a975d3752f8a7ad077fc6419323 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 21:57:36 +0800 Subject: [PATCH 85/91] logic fix --- src/neo/Network/P2P/TaskManager.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index 0fab0e446e..c7596fd66e 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -352,10 +352,10 @@ private void RequestTasks(IActorRef remoteNode, TaskSession session) { uint startHeight = currentHeight; while (globalIndexTasks.ContainsKey(++startHeight)) { } - if (startHeight > session.LastBlockIndex) return; + if (startHeight > session.LastBlockIndex || startHeight >= currentHeight + InvPayload.MaxHashesCount) return; uint endHeight = startHeight; - while (!globalIndexTasks.ContainsKey(++endHeight) && endHeight <= session.LastBlockIndex) { } - uint count = Math.Min(endHeight - startHeight, HeadersPayload.MaxHeadersCount); + while (!globalIndexTasks.ContainsKey(++endHeight) && endHeight <= session.LastBlockIndex && endHeight <= currentHeight + InvPayload.MaxHashesCount) { } + uint count = Math.Min(endHeight - startHeight, InvPayload.MaxHashesCount); for (uint i = 0; i < count; i++) { session.IndexTasks[startHeight + i] = TimeProvider.Current.UtcNow; From 7057ec781ad489cd5d349abc33b6d1a53fff4b5d Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 22:14:12 +0800 Subject: [PATCH 86/91] fix --- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 8f3936550a..9c3749d415 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -298,8 +298,8 @@ private void OnInventoryReceived(IInventory inventory) pendingKnownHashes.Remove(inventory.Hash); if (inventory is Block block) { - if (block.Index > NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View) + InvPayload.MaxHashesCount) return; UpdateLastBlockIndex(block.Index); + if (block.Index > NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View) + InvPayload.MaxHashesCount) return; } knownHashes.Add(inventory.Hash); system.TaskManager.Tell(inventory); From 9d8057b656d418ec59f576fbcd43c9c9a047dd5a Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Tue, 2 Feb 2021 22:30:09 +0800 Subject: [PATCH 87/91] fix --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c7596fd66e..d0dab74cc3 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -74,7 +74,7 @@ private void OnNewTasks(InvPayload payload) // Do not accept payload of type InventoryType.TX if not synced on HeaderHeight uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; - if (payload.Type == InventoryType.TX && currentHeight < headerHeight) + if (currentHeight < headerHeight) { RequestTasks(Sender, session); return; From a84ce4bb9cbd045fff16d6c3efc0061de950ccdd Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Wed, 3 Feb 2021 10:15:39 +0800 Subject: [PATCH 88/91] Revert "fix" This reverts commit 9d8057b656d418ec59f576fbcd43c9c9a047dd5a. --- src/neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index d0dab74cc3..c7596fd66e 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -74,7 +74,7 @@ private void OnNewTasks(InvPayload payload) // Do not accept payload of type InventoryType.TX if not synced on HeaderHeight uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; - if (currentHeight < headerHeight) + if (payload.Type == InventoryType.TX && currentHeight < headerHeight) { RequestTasks(Sender, session); return; From 954a04c3914a2b4d57f5eb68d1ea7e1ccfdc3bd4 Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Wed, 3 Feb 2021 10:50:28 +0800 Subject: [PATCH 89/91] logic fix --- src/neo/Network/P2P/TaskManager.cs | 4 ++-- src/neo/Network/P2P/TaskSession.cs | 2 +- tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/neo/Network/P2P/TaskManager.cs b/src/neo/Network/P2P/TaskManager.cs index c7596fd66e..8ea8457a72 100644 --- a/src/neo/Network/P2P/TaskManager.cs +++ b/src/neo/Network/P2P/TaskManager.cs @@ -74,7 +74,7 @@ private void OnNewTasks(InvPayload payload) // Do not accept payload of type InventoryType.TX if not synced on HeaderHeight uint currentHeight = NativeContract.Ledger.CurrentIndex(Blockchain.Singleton.View); uint headerHeight = Blockchain.Singleton.HeaderCache.Last?.Index ?? currentHeight; - if (payload.Type == InventoryType.TX && currentHeight < headerHeight) + if (currentHeight < headerHeight && (payload.Type == InventoryType.TX || (payload.Type == InventoryType.Block && currentHeight < session.LastBlockIndex - InvPayload.MaxHashesCount))) { RequestTasks(Sender, session); return; @@ -311,7 +311,7 @@ public static Props Props(NeoSystem system) private void RequestTasks(IActorRef remoteNode, TaskSession session) { - if (session.HasTask) return; + if (session.HasTooManyTasks) return; DataCache snapshot = Blockchain.Singleton.View; diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index cd4a45ea80..b089ce895d 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -12,7 +12,7 @@ internal class TaskSession public Dictionary IndexTasks { get; } = new Dictionary(); public HashSet AvailableTasks { get; } = new HashSet(); public Dictionary ReceivedBlock { get; } = new Dictionary(); - public bool HasTask => InvTasks.Count > 0 || IndexTasks.Count > 0; + public bool HasTooManyTasks => InvTasks.Count > 100 || IndexTasks.Count > 0; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } public bool MempoolSent { get; set; } diff --git a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs index 7ebc57fc07..bee2bc8efe 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_TaskSession.cs @@ -15,14 +15,14 @@ public void CreateTest() { var ses = new TaskSession(new VersionPayload() { Capabilities = new NodeCapability[] { new FullNodeCapability(123) } }); - Assert.IsFalse(ses.HasTask); + Assert.IsFalse(ses.HasTooManyTasks); Assert.AreEqual((uint)123, ses.LastBlockIndex); Assert.AreEqual(0, ses.IndexTasks.Count); Assert.IsTrue(ses.IsFullNode); ses = new TaskSession(new VersionPayload() { Capabilities = Array.Empty() }); - Assert.IsFalse(ses.HasTask); + Assert.IsFalse(ses.HasTooManyTasks); Assert.AreEqual((uint)0, ses.LastBlockIndex); Assert.AreEqual(0, ses.IndexTasks.Count); Assert.IsFalse(ses.IsFullNode); From 6d0814c36c814c8a88f60e7f51d43030bea0db2d Mon Sep 17 00:00:00 2001 From: Qiao Jin Date: Wed, 3 Feb 2021 11:10:19 +0800 Subject: [PATCH 90/91] code clean --- src/neo/Persistence/IReadOnlyStore.cs | 1 - src/neo/Persistence/MemorySnapshot.cs | 1 - src/neo/Persistence/MemoryStore.cs | 1 - src/neo/SmartContract/Iterators/StorageIterator.cs | 1 - src/neo/SmartContract/KeyBuilder.cs | 1 - tests/neo.UnitTests/Persistence/UT_MemoryStore.cs | 1 - .../neo.UnitTests/SmartContract/Iterators/UT_StorageIterator.cs | 1 - tests/neo.UnitTests/UT_DataCache.cs | 1 - 8 files changed, 8 deletions(-) diff --git a/src/neo/Persistence/IReadOnlyStore.cs b/src/neo/Persistence/IReadOnlyStore.cs index f13fd56d7f..7200691084 100644 --- a/src/neo/Persistence/IReadOnlyStore.cs +++ b/src/neo/Persistence/IReadOnlyStore.cs @@ -1,4 +1,3 @@ -using Neo.IO.Caching; using System.Collections.Generic; namespace Neo.Persistence diff --git a/src/neo/Persistence/MemorySnapshot.cs b/src/neo/Persistence/MemorySnapshot.cs index 7b3c77dbe8..9c2ee79998 100644 --- a/src/neo/Persistence/MemorySnapshot.cs +++ b/src/neo/Persistence/MemorySnapshot.cs @@ -1,5 +1,4 @@ using Neo.IO; -using Neo.IO.Caching; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Immutable; diff --git a/src/neo/Persistence/MemoryStore.cs b/src/neo/Persistence/MemoryStore.cs index 705f509b87..1cf51dde51 100644 --- a/src/neo/Persistence/MemoryStore.cs +++ b/src/neo/Persistence/MemoryStore.cs @@ -1,5 +1,4 @@ using Neo.IO; -using Neo.IO.Caching; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; diff --git a/src/neo/SmartContract/Iterators/StorageIterator.cs b/src/neo/SmartContract/Iterators/StorageIterator.cs index bcca51b471..74dc5015cc 100644 --- a/src/neo/SmartContract/Iterators/StorageIterator.cs +++ b/src/neo/SmartContract/Iterators/StorageIterator.cs @@ -1,4 +1,3 @@ -using Neo.Ledger; using Neo.VM; using Neo.VM.Types; using System.Collections.Generic; diff --git a/src/neo/SmartContract/KeyBuilder.cs b/src/neo/SmartContract/KeyBuilder.cs index c725da370b..80bea0e783 100644 --- a/src/neo/SmartContract/KeyBuilder.cs +++ b/src/neo/SmartContract/KeyBuilder.cs @@ -1,5 +1,4 @@ using Neo.IO; -using Neo.Ledger; using System; using System.IO; diff --git a/tests/neo.UnitTests/Persistence/UT_MemoryStore.cs b/tests/neo.UnitTests/Persistence/UT_MemoryStore.cs index 5feff2b677..23ac57e261 100644 --- a/tests/neo.UnitTests/Persistence/UT_MemoryStore.cs +++ b/tests/neo.UnitTests/Persistence/UT_MemoryStore.cs @@ -1,5 +1,4 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.IO.Caching; using Neo.Persistence; using System.Linq; diff --git a/tests/neo.UnitTests/SmartContract/Iterators/UT_StorageIterator.cs b/tests/neo.UnitTests/SmartContract/Iterators/UT_StorageIterator.cs index 806c4d4817..a9ccec7769 100644 --- a/tests/neo.UnitTests/SmartContract/Iterators/UT_StorageIterator.cs +++ b/tests/neo.UnitTests/SmartContract/Iterators/UT_StorageIterator.cs @@ -1,6 +1,5 @@ using FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.Ledger; using Neo.SmartContract; using Neo.SmartContract.Iterators; using Neo.VM.Types; diff --git a/tests/neo.UnitTests/UT_DataCache.cs b/tests/neo.UnitTests/UT_DataCache.cs index b10747df8d..ca15234c9b 100644 --- a/tests/neo.UnitTests/UT_DataCache.cs +++ b/tests/neo.UnitTests/UT_DataCache.cs @@ -1,5 +1,4 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.IO.Caching; using Neo.Ledger; using Neo.Persistence; using Neo.SmartContract; From 4a5ebbb63b9aff11388a75099d469a6374672735 Mon Sep 17 00:00:00 2001 From: Qiao Jin <43407364+Qiao-Jin@users.noreply.github.com> Date: Wed, 3 Feb 2021 12:38:18 +0800 Subject: [PATCH 91/91] Update src/neo/Network/P2P/TaskSession.cs Co-authored-by: Erik Zhang --- src/neo/Network/P2P/TaskSession.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Network/P2P/TaskSession.cs b/src/neo/Network/P2P/TaskSession.cs index b089ce895d..6764789b24 100644 --- a/src/neo/Network/P2P/TaskSession.cs +++ b/src/neo/Network/P2P/TaskSession.cs @@ -12,7 +12,7 @@ internal class TaskSession public Dictionary IndexTasks { get; } = new Dictionary(); public HashSet AvailableTasks { get; } = new HashSet(); public Dictionary ReceivedBlock { get; } = new Dictionary(); - public bool HasTooManyTasks => InvTasks.Count > 100 || IndexTasks.Count > 0; + public bool HasTooManyTasks => InvTasks.Count + IndexTasks.Count >= 100; public bool IsFullNode { get; } public uint LastBlockIndex { get; set; } public bool MempoolSent { get; set; }