From b7f9c026258a70a219a3e87bafabf43249a38796 Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Tue, 22 Oct 2019 16:29:32 +0800 Subject: [PATCH 01/24] Replace ExceptWith & UnionWith with equal but faster functionality --- neo/Network/P2P/Payloads/Transaction.cs | 4 ++-- neo/Network/P2P/TaskManager.cs | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/neo/Network/P2P/Payloads/Transaction.cs b/neo/Network/P2P/Payloads/Transaction.cs index af38ef6dcc..79fd1981ca 100644 --- a/neo/Network/P2P/Payloads/Transaction.cs +++ b/neo/Network/P2P/Payloads/Transaction.cs @@ -125,8 +125,8 @@ public override int GetHashCode() public UInt160[] GetScriptHashesForVerifying(Snapshot snapshot) { - var hashes = new HashSet { Sender }; - hashes.UnionWith(Cosigners.Select(p => p.Account)); + var hashes = new HashSet(Cosigners.Select(p => p.Account)); + hashes.UnionWith(new HashSet { Sender }); return hashes.OrderBy(p => p).ToArray(); } diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index 2c0f7ae07b..3bed0c527b 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -58,11 +58,11 @@ private void OnNewTasks(InvPayload payload) return; } HashSet hashes = new HashSet(payload.Hashes); - hashes.ExceptWith(knownHashes); + hashes.RemoveWhere(q => knownHashes.Contains(q)); if (payload.Type == InventoryType.Block) session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p))); - hashes.ExceptWith(globalTasks.Keys); + hashes.RemoveWhere(q => globalTasks.Keys.Contains(q)); if (hashes.Count == 0) { RequestTasks(session); @@ -203,7 +203,7 @@ private void RequestTasks(TaskSession session) if (session.HasTask) return; if (session.AvailableTasks.Count > 0) { - session.AvailableTasks.ExceptWith(knownHashes); + session.AvailableTasks.RemoveWhere(q => knownHashes.Contains(q)); session.AvailableTasks.RemoveWhere(p => Blockchain.Singleton.ContainsBlock(p)); HashSet hashes = new HashSet(session.AvailableTasks); if (hashes.Count > 0) From 318c99ff1c57956b5c84d7dc470f3f5ccccccd3d Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 22 Oct 2019 11:35:29 +0200 Subject: [PATCH 02/24] Optimization --- neo/Network/P2P/Payloads/Transaction.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neo/Network/P2P/Payloads/Transaction.cs b/neo/Network/P2P/Payloads/Transaction.cs index 79fd1981ca..3a6ea6eb85 100644 --- a/neo/Network/P2P/Payloads/Transaction.cs +++ b/neo/Network/P2P/Payloads/Transaction.cs @@ -125,6 +125,8 @@ public override int GetHashCode() public UInt160[] GetScriptHashesForVerifying(Snapshot snapshot) { + if (Cosigners.Length == 0) return new UInt160[] { Sender }; + var hashes = new HashSet(Cosigners.Select(p => p.Account)); hashes.UnionWith(new HashSet { Sender }); return hashes.OrderBy(p => p).ToArray(); From af23059d5515f7d2a54d4ea731bb02bc94810a91 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 22 Oct 2019 12:38:54 +0200 Subject: [PATCH 03/24] Optimization --- neo/Network/P2P/Payloads/Transaction.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/neo/Network/P2P/Payloads/Transaction.cs b/neo/Network/P2P/Payloads/Transaction.cs index 3a6ea6eb85..11e703a43b 100644 --- a/neo/Network/P2P/Payloads/Transaction.cs +++ b/neo/Network/P2P/Payloads/Transaction.cs @@ -125,7 +125,10 @@ public override int GetHashCode() public UInt160[] GetScriptHashesForVerifying(Snapshot snapshot) { - if (Cosigners.Length == 0) return new UInt160[] { Sender }; + if (Cosigners.Length == 0 || (Cosigners.Length == 1 && Cosigners[0].Account == Sender)) + { + return new UInt160[] { Sender }; + } var hashes = new HashSet(Cosigners.Select(p => p.Account)); hashes.UnionWith(new HashSet { Sender }); From 82930daac7a4361f8177339833572cfecda8b954 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 22 Oct 2019 12:47:14 +0200 Subject: [PATCH 04/24] Optimize remove --- neo/Helper.cs | 12 ++++++++++++ neo/Network/P2P/TaskManager.cs | 8 ++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 1a513ec79e..4f7db112a2 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -54,6 +54,18 @@ internal static int GetLowestSetBit(this BigInteger i) throw new Exception(); } + internal static void Remove(this HashSet set, IEnumerable remove) + { + if (set.Count > 1000) + { + set.ExceptWith(remove); + } + else + { + set.RemoveWhere(u => remove.Contains(u)); + } + } + internal static string GetVersion(this Assembly assembly) { CustomAttributeData attribute = assembly.CustomAttributes.FirstOrDefault(p => p.AttributeType == typeof(AssemblyInformationalVersionAttribute)); diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index 3bed0c527b..b83b20cb0e 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -58,11 +58,11 @@ private void OnNewTasks(InvPayload payload) return; } HashSet hashes = new HashSet(payload.Hashes); - hashes.RemoveWhere(q => knownHashes.Contains(q)); + hashes.Remove(knownHashes); if (payload.Type == InventoryType.Block) session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p))); - hashes.RemoveWhere(q => globalTasks.Keys.Contains(q)); + hashes.Remove(globalTasks.Keys); if (hashes.Count == 0) { RequestTasks(session); @@ -203,7 +203,7 @@ private void RequestTasks(TaskSession session) if (session.HasTask) return; if (session.AvailableTasks.Count > 0) { - session.AvailableTasks.RemoveWhere(q => knownHashes.Contains(q)); + session.AvailableTasks.Remove(knownHashes); session.AvailableTasks.RemoveWhere(p => Blockchain.Singleton.ContainsBlock(p)); HashSet hashes = new HashSet(session.AvailableTasks); if (hashes.Count > 0) @@ -213,7 +213,7 @@ private void RequestTasks(TaskSession session) if (!IncrementGlobalTask(hash)) hashes.Remove(hash); } - session.AvailableTasks.ExceptWith(hashes); + session.AvailableTasks.Remove(hashes); foreach (UInt256 hash in hashes) session.Tasks[hash] = DateTime.UtcNow; foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray())) From 54d2af86c1de9469fbd879efaba1208d843c5e88 Mon Sep 17 00:00:00 2001 From: Qiao-Jin <43407364+Qiao-Jin@users.noreply.github.com> Date: Wed, 23 Oct 2019 10:25:04 +0800 Subject: [PATCH 05/24] Update neo/Network/P2P/TaskManager.cs Co-Authored-By: Erik Zhang --- neo/Network/P2P/TaskManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index 3bed0c527b..1736036688 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -62,7 +62,7 @@ private void OnNewTasks(InvPayload payload) if (payload.Type == InventoryType.Block) session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p))); - hashes.RemoveWhere(q => globalTasks.Keys.Contains(q)); + hashes.RemoveWhere(q => globalTasks.ContainsKey(q)); if (hashes.Count == 0) { RequestTasks(session); From a745880494cf1c3aa6e48b0ffd391384f12c94dd Mon Sep 17 00:00:00 2001 From: Jin Qiao Date: Wed, 23 Oct 2019 10:30:40 +0800 Subject: [PATCH 06/24] Code optimization --- neo/Network/P2P/Payloads/Transaction.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/neo/Network/P2P/Payloads/Transaction.cs b/neo/Network/P2P/Payloads/Transaction.cs index 11e703a43b..af38ef6dcc 100644 --- a/neo/Network/P2P/Payloads/Transaction.cs +++ b/neo/Network/P2P/Payloads/Transaction.cs @@ -125,13 +125,8 @@ public override int GetHashCode() public UInt160[] GetScriptHashesForVerifying(Snapshot snapshot) { - if (Cosigners.Length == 0 || (Cosigners.Length == 1 && Cosigners[0].Account == Sender)) - { - return new UInt160[] { Sender }; - } - - var hashes = new HashSet(Cosigners.Select(p => p.Account)); - hashes.UnionWith(new HashSet { Sender }); + var hashes = new HashSet { Sender }; + hashes.UnionWith(Cosigners.Select(p => p.Account)); return hashes.OrderBy(p => p).ToArray(); } From a84eff9a7b8c240a749e3e9eb64d31d9d3a296ca Mon Sep 17 00:00:00 2001 From: Qiao-Jin <43407364+Qiao-Jin@users.noreply.github.com> Date: Wed, 23 Oct 2019 11:06:42 +0800 Subject: [PATCH 07/24] Update Helper.cs --- neo/Helper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 4f7db112a2..6f3ae05c6d 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -56,7 +56,7 @@ internal static int GetLowestSetBit(this BigInteger i) internal static void Remove(this HashSet set, IEnumerable remove) { - if (set.Count > 1000) + if (set.Count > 600000) { set.ExceptWith(remove); } From 089b935236584e0da595ae416cd9d33101a3ec47 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 09:11:59 +0200 Subject: [PATCH 08/24] Small change --- neo/Helper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 6f3ae05c6d..4185ad327c 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -56,7 +56,7 @@ internal static int GetLowestSetBit(this BigInteger i) internal static void Remove(this HashSet set, IEnumerable remove) { - if (set.Count > 600000) + if (set.Count > 600_000) { set.ExceptWith(remove); } From a571c22242e766042e99c2d74ad655020882fb79 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 09:52:13 +0200 Subject: [PATCH 09/24] Optimization --- neo/Helper.cs | 12 ++++++++++++ neo/Network/P2P/TaskManager.cs | 2 +- neo/UIntBase.cs | 7 ++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 4185ad327c..37e2d5d5e2 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -66,6 +66,18 @@ internal static void Remove(this HashSet set, IEnumerable remove) } } + internal static void Remove(this HashSet set, IDictionary remove) + { + if (set.Count > 600_000) + { + set.ExceptWith(remove.Keys); + } + else + { + set.RemoveWhere(u => remove.ContainsKey(u)); + } + } + internal static string GetVersion(this Assembly assembly) { CustomAttributeData attribute = assembly.CustomAttributes.FirstOrDefault(p => p.AttributeType == typeof(AssemblyInformationalVersionAttribute)); diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs index b83b20cb0e..82e2e9b87a 100644 --- a/neo/Network/P2P/TaskManager.cs +++ b/neo/Network/P2P/TaskManager.cs @@ -62,7 +62,7 @@ private void OnNewTasks(InvPayload payload) if (payload.Type == InventoryType.Block) session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p))); - hashes.Remove(globalTasks.Keys); + hashes.Remove(globalTasks); if (hashes.Count == 0) { RequestTasks(session); diff --git a/neo/UIntBase.cs b/neo/UIntBase.cs index af26e36d0a..b0372a6e77 100644 --- a/neo/UIntBase.cs +++ b/neo/UIntBase.cs @@ -61,7 +61,12 @@ public bool Equals(UIntBase other) return true; if (data_bytes.Length != other.data_bytes.Length) return false; - return data_bytes.SequenceEqual(other.data_bytes); + + for (int x = data_bytes.Length - 1; x >= 0; x--) + { + if (data_bytes[x] != other.data_bytes[x]) return false; + } + return true; } /// From 4381b4c7ed34d003a4451b7a4ee55de0d27a10c2 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:05:11 +0200 Subject: [PATCH 10/24] Update Helper.cs --- neo/Helper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 4185ad327c..0d64b69fe3 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -54,7 +54,7 @@ internal static int GetLowestSetBit(this BigInteger i) throw new Exception(); } - internal static void Remove(this HashSet set, IEnumerable remove) + internal static void Remove(this HashSet set, FIFOSet remove) { if (set.Count > 600_000) { From dd4a435aaf8311553dc773016d01afd4181a491a Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:06:11 +0200 Subject: [PATCH 11/24] Revert --- neo/Helper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 0d64b69fe3..4185ad327c 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -54,7 +54,7 @@ internal static int GetLowestSetBit(this BigInteger i) throw new Exception(); } - internal static void Remove(this HashSet set, FIFOSet remove) + internal static void Remove(this HashSet set, IEnumerable remove) { if (set.Count > 600_000) { From 95136ede25bdf97954e1313819eb1a31eff3e047 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:12:13 +0200 Subject: [PATCH 12/24] Optimization --- neo/Helper.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 37e2d5d5e2..734b639a14 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -1,4 +1,5 @@ using Microsoft.Extensions.Configuration; +using Neo.IO.Caching; using Neo.Plugins; using System; using System.Collections.Generic; @@ -54,7 +55,20 @@ internal static int GetLowestSetBit(this BigInteger i) throw new Exception(); } - internal static void Remove(this HashSet set, IEnumerable remove) + internal static void Remove(this HashSet set, HashSet remove) + { + if (set.Count > 600_000) + { + set.ExceptWith(remove); + } + else + { + set.RemoveWhere(u => remove.Contains(u)); + } + } + + internal static void Remove(this HashSet set, FIFOSet remove) + where T : IEquatable { if (set.Count > 600_000) { From edc0333a1235ea1a1cddfae765ca0298e305cc58 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:28:03 +0200 Subject: [PATCH 13/24] Optimize FIFOSet --- neo/IO/Caching/FIFOSet.cs | 105 ++++++++++++++++++++++++++++++++------ 1 file changed, 90 insertions(+), 15 deletions(-) diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index 98733ce444..b6947f904c 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -1,16 +1,30 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Collections.Specialized; -using System.Linq; namespace Neo.IO.Caching { internal class FIFOSet : IEnumerable where T : IEquatable { + class Entry + { + public readonly T Value; + public Entry Next; + + public Entry(T current) + { + Value = current; + } + } + private readonly int maxCapacity; private readonly int removeCount; - private readonly OrderedDictionary dictionary; + + private int _count = 0; + private Entry _firstEntry = null; + private Entry _last = null; + + public int Count => _count; public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) { @@ -19,45 +33,106 @@ public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) this.maxCapacity = maxCapacity; this.removeCount = Math.Max((int)(maxCapacity * batchSize), 1); - this.dictionary = new OrderedDictionary(maxCapacity); } public bool Add(T item) { - if (dictionary.Contains(item)) return false; - if (dictionary.Count >= maxCapacity) + if (Contains(item)) return false; + if (Count >= maxCapacity) { if (removeCount == maxCapacity) { - dictionary.Clear(); + _last = _firstEntry = null; } else { for (int i = 0; i < removeCount; i++) - dictionary.RemoveAt(0); + { + RemoveFirst(); + } } } - dictionary.Add(item, null); + + if (_last == null) + { + _firstEntry = _last = new Entry(item); + } + else + { + _last = _last.Next = new Entry(item); + } + + _count++; return true; } + private void RemoveFirst() + { + if (_firstEntry != null) + { + _firstEntry = _firstEntry.Next; + _count--; + } + } + public bool Contains(T item) { - return dictionary.Contains(item); + var current = _firstEntry; + + while (current != null) + { + if (current.Value.Equals(item)) return true; + current = current.Next; + } + + return false; + } + + private void Remove(T item) + { + var prev = _firstEntry; + var current = _firstEntry; + + while (current != null) + { + if (current.Value.Equals(item)) + { + // First + + if (prev == null) + { + _firstEntry = current.Next; + return; + } + else + { + prev.Next = current.Next; + return; + } + } + + prev = current; + current = current.Next; + } } - public void ExceptWith(IEnumerable hashes) + public void ExceptWith(IEnumerable entries) { - foreach (var hash in hashes) + foreach (var entry in entries) { - dictionary.Remove(hash); + Remove(entry); } } public IEnumerator GetEnumerator() { - var entries = dictionary.Keys.Cast().ToArray(); - foreach (var entry in entries) yield return entry; + var current = _firstEntry; + + while (current != null) + { + yield return current.Value; + current = current.Next; + } } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); From 4ddc299924bef295928b318540b0de2bd3060c1c Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:32:47 +0200 Subject: [PATCH 14/24] Rename --- neo/IO/Caching/FIFOSet.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index b6947f904c..912a4e3805 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -22,7 +22,7 @@ public Entry(T current) private int _count = 0; private Entry _firstEntry = null; - private Entry _last = null; + private Entry _lastEntry = null; public int Count => _count; @@ -42,7 +42,7 @@ public bool Add(T item) { if (removeCount == maxCapacity) { - _last = _firstEntry = null; + _lastEntry = _firstEntry = null; } else { @@ -53,13 +53,13 @@ public bool Add(T item) } } - if (_last == null) + if (_lastEntry == null) { - _firstEntry = _last = new Entry(item); + _firstEntry = _lastEntry = new Entry(item); } else { - _last = _last.Next = new Entry(item); + _lastEntry = _lastEntry.Next = new Entry(item); } _count++; From 6da2fc237c7ee780190b48c3ce193def47b72477 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:34:24 +0200 Subject: [PATCH 15/24] Inline --- neo/UIntBase.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neo/UIntBase.cs b/neo/UIntBase.cs index b0372a6e77..90490ac369 100644 --- a/neo/UIntBase.cs +++ b/neo/UIntBase.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; namespace Neo { @@ -115,6 +116,7 @@ void ISerializable.Serialize(BinaryWriter writer) /// /// Method ToArray() returns the byte array data_bytes, which stores the little-endian unsigned int /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] public byte[] ToArray() { return data_bytes; From 88753a6d94bf6902929492ec3bc214bb00ddc8c9 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:39:17 +0200 Subject: [PATCH 16/24] Update UT_FIFOSet.cs --- neo.UnitTests/IO/Caching/UT_FIFOSet.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/neo.UnitTests/IO/Caching/UT_FIFOSet.cs b/neo.UnitTests/IO/Caching/UT_FIFOSet.cs index e6e01710d9..3187438a5c 100644 --- a/neo.UnitTests/IO/Caching/UT_FIFOSet.cs +++ b/neo.UnitTests/IO/Caching/UT_FIFOSet.cs @@ -144,6 +144,21 @@ public void TestExceptWith() }; set.ExceptWith(new UInt256[] { b, c }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a }); + + set = new FIFOSet(10) + { + b, + c + }; + set.ExceptWith(new UInt256[] { b, c }); + CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c }); + + set = new FIFOSet(10) + { + c + }; + set.ExceptWith(new UInt256[] { b, c }); + CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, b }); } } } From 59ee1205981e749bce425de7b28b7b39cdc320e9 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:46:08 +0200 Subject: [PATCH 17/24] Fix --- neo/IO/Caching/FIFOSet.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index 912a4e3805..5a3cccbfc9 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -102,11 +102,20 @@ private void Remove(T item) if (prev == null) { _firstEntry = current.Next; + _count--; return; } else { - prev.Next = current.Next; + if (current == _firstEntry) + { + _firstEntry = prev.Next; + } + else + { + prev.Next = current.Next; + } + _count--; return; } } From 921a3439370c8f014814a07fe802ecd8eed789ef Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:46:34 +0200 Subject: [PATCH 18/24] Update UT_FIFOSet.cs --- neo.UnitTests/IO/Caching/UT_FIFOSet.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/neo.UnitTests/IO/Caching/UT_FIFOSet.cs b/neo.UnitTests/IO/Caching/UT_FIFOSet.cs index 3187438a5c..d785127778 100644 --- a/neo.UnitTests/IO/Caching/UT_FIFOSet.cs +++ b/neo.UnitTests/IO/Caching/UT_FIFOSet.cs @@ -147,17 +147,20 @@ public void TestExceptWith() set = new FIFOSet(10) { + a, b, c }; - set.ExceptWith(new UInt256[] { b, c }); + set.ExceptWith(new UInt256[] { a }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c }); set = new FIFOSet(10) { + a, + b, c }; - set.ExceptWith(new UInt256[] { b, c }); + set.ExceptWith(new UInt256[] { c }); CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, b }); } } From 621c4e3ba6cd05fbd8c43a523e70296580c033d4 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 10:48:51 +0200 Subject: [PATCH 19/24] Update FIFOSet.cs --- neo/IO/Caching/FIFOSet.cs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index 5a3cccbfc9..ab6ff1512c 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -97,13 +97,10 @@ private void Remove(T item) { if (current.Value.Equals(item)) { - // First - if (prev == null) { + // First entry _firstEntry = current.Next; - _count--; - return; } else { @@ -115,9 +112,9 @@ private void Remove(T item) { prev.Next = current.Next; } - _count--; - return; } + _count--; + return; } prev = current; From 051b7c1f51b89edb2fbbc22e1e9e6c3fced820cf Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 23 Oct 2019 11:42:18 +0200 Subject: [PATCH 20/24] Update FIFOSet.cs --- neo/IO/Caching/FIFOSet.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index ab6ff1512c..172abdb7fe 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -43,6 +43,7 @@ public bool Add(T item) if (removeCount == maxCapacity) { _lastEntry = _firstEntry = null; + _count = 0; } else { From f1cef9635ce38d40cf2ac38f3a9f462c050c5669 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 30 Oct 2019 18:57:24 +0800 Subject: [PATCH 21/24] Revert FIFOSet --- neo/IO/Caching/FIFOSet.cs | 108 +++++--------------------------------- 1 file changed, 13 insertions(+), 95 deletions(-) diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index 172abdb7fe..5a066c1ce3 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -1,30 +1,16 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Collections.Specialized; +using System.Linq; namespace Neo.IO.Caching { internal class FIFOSet : IEnumerable where T : IEquatable { - class Entry - { - public readonly T Value; - public Entry Next; - - public Entry(T current) - { - Value = current; - } - } - private readonly int maxCapacity; private readonly int removeCount; - - private int _count = 0; - private Entry _firstEntry = null; - private Entry _lastEntry = null; - - public int Count => _count; + private readonly OrderedDictionary dictionary; public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) { @@ -33,113 +19,45 @@ public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) this.maxCapacity = maxCapacity; this.removeCount = Math.Max((int)(maxCapacity * batchSize), 1); + this.dictionary = new OrderedDictionary(maxCapacity); } public bool Add(T item) { - if (Contains(item)) return false; - if (Count >= maxCapacity) + if (dictionary.Contains(item)) return false; + if (dictionary.Count >= maxCapacity) { if (removeCount == maxCapacity) { - _lastEntry = _firstEntry = null; - _count = 0; + dictionary.Clear(); } else { for (int i = 0; i < removeCount; i++) - { - RemoveFirst(); - } + dictionary.RemoveAt(0); } } - - if (_lastEntry == null) - { - _firstEntry = _lastEntry = new Entry(item); - } - else - { - _lastEntry = _lastEntry.Next = new Entry(item); - } - - _count++; + dictionary.Add(item, null); return true; } - private void RemoveFirst() - { - if (_firstEntry != null) - { - _firstEntry = _firstEntry.Next; - _count--; - } - } - public bool Contains(T item) { - var current = _firstEntry; - - while (current != null) - { - if (current.Value.Equals(item)) return true; - current = current.Next; - } - - return false; - } - - private void Remove(T item) - { - var prev = _firstEntry; - var current = _firstEntry; - - while (current != null) - { - if (current.Value.Equals(item)) - { - if (prev == null) - { - // First entry - _firstEntry = current.Next; - } - else - { - if (current == _firstEntry) - { - _firstEntry = prev.Next; - } - else - { - prev.Next = current.Next; - } - } - _count--; - return; - } - - prev = current; - current = current.Next; - } + return dictionary.Contains(item); } public void ExceptWith(IEnumerable entries) { foreach (var entry in entries) { - Remove(entry); + dictionary.Remove(entry); } } public IEnumerator GetEnumerator() { - var current = _firstEntry; - - while (current != null) - { - yield return current.Value; - current = current.Next; - } + var entries = dictionary.Keys.Cast().ToArray(); + foreach (var entry in entries) yield return entry; } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); From 2e704e6033e7fe38bb8ec32441bacbda84804cda Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 30 Oct 2019 19:09:38 +0800 Subject: [PATCH 22/24] Update Helper.cs --- neo/Helper.cs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 734b639a14..6fb28ed8df 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -1,6 +1,5 @@ using Microsoft.Extensions.Configuration; using Neo.IO.Caching; -using Neo.Plugins; using System; using System.Collections.Generic; using System.Globalization; @@ -55,40 +54,40 @@ internal static int GetLowestSetBit(this BigInteger i) throw new Exception(); } - internal static void Remove(this HashSet set, HashSet remove) + internal static void Remove(this HashSet set, ISet other) { if (set.Count > 600_000) { - set.ExceptWith(remove); + set.ExceptWith(other); } else { - set.RemoveWhere(u => remove.Contains(u)); + set.RemoveWhere(u => other.Contains(u)); } } - internal static void Remove(this HashSet set, FIFOSet remove) + internal static void Remove(this HashSet set, FIFOSet other) where T : IEquatable { if (set.Count > 600_000) { - set.ExceptWith(remove); + set.ExceptWith(other); } else { - set.RemoveWhere(u => remove.Contains(u)); + set.RemoveWhere(u => other.Contains(u)); } } - internal static void Remove(this HashSet set, IDictionary remove) + internal static void Remove(this HashSet set, IReadOnlyDictionary other) { if (set.Count > 600_000) { - set.ExceptWith(remove.Keys); + set.ExceptWith(other.Keys); } else { - set.RemoveWhere(u => remove.ContainsKey(u)); + set.RemoveWhere(u => other.ContainsKey(u)); } } From 12aa262ea9d6af22170480141b401e1f814c2832 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 30 Oct 2019 19:12:04 +0800 Subject: [PATCH 23/24] Optimize --- neo/Helper.cs | 6 +++--- neo/IO/Caching/FIFOSet.cs | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/neo/Helper.cs b/neo/Helper.cs index 6fb28ed8df..fa3b384a35 100644 --- a/neo/Helper.cs +++ b/neo/Helper.cs @@ -56,7 +56,7 @@ internal static int GetLowestSetBit(this BigInteger i) internal static void Remove(this HashSet set, ISet other) { - if (set.Count > 600_000) + if (set.Count > other.Count) { set.ExceptWith(other); } @@ -69,7 +69,7 @@ internal static void Remove(this HashSet set, ISet other) internal static void Remove(this HashSet set, FIFOSet other) where T : IEquatable { - if (set.Count > 600_000) + if (set.Count > other.Count) { set.ExceptWith(other); } @@ -81,7 +81,7 @@ internal static void Remove(this HashSet set, FIFOSet other) internal static void Remove(this HashSet set, IReadOnlyDictionary other) { - if (set.Count > 600_000) + if (set.Count > other.Count) { set.ExceptWith(other.Keys); } diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs index 5a066c1ce3..af65db1b8b 100644 --- a/neo/IO/Caching/FIFOSet.cs +++ b/neo/IO/Caching/FIFOSet.cs @@ -6,12 +6,14 @@ namespace Neo.IO.Caching { - internal class FIFOSet : IEnumerable where T : IEquatable + internal class FIFOSet : IReadOnlyCollection where T : IEquatable { private readonly int maxCapacity; private readonly int removeCount; private readonly OrderedDictionary dictionary; + public int Count => dictionary.Count; + public FIFOSet(int maxCapacity, decimal batchSize = 0.1m) { if (maxCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(maxCapacity)); From 360f20aacb6891c3285e261dcb06be6bf09cb7bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Thu, 31 Oct 2019 10:04:00 -0300 Subject: [PATCH 24/24] Reverting independet byte checks to SequenceEqual --- neo/UIntBase.cs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/neo/UIntBase.cs b/neo/UIntBase.cs index 90490ac369..6b5f6d6acd 100644 --- a/neo/UIntBase.cs +++ b/neo/UIntBase.cs @@ -62,12 +62,7 @@ public bool Equals(UIntBase other) return true; if (data_bytes.Length != other.data_bytes.Length) return false; - - for (int x = data_bytes.Length - 1; x >= 0; x--) - { - if (data_bytes[x] != other.data_bytes[x]) return false; - } - return true; + return data_bytes.SequenceEqual(other.data_bytes); } ///