diff --git a/neo.UnitTests/IO/Caching/UT_FIFOSet.cs b/neo.UnitTests/IO/Caching/UT_FIFOSet.cs
index e6e01710d9..d785127778 100644
--- a/neo.UnitTests/IO/Caching/UT_FIFOSet.cs
+++ b/neo.UnitTests/IO/Caching/UT_FIFOSet.cs
@@ -144,6 +144,24 @@ public void TestExceptWith()
             };
             set.ExceptWith(new UInt256[] { b, c });
             CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a });
+
+            set = new FIFOSet<UInt256>(10)
+            {
+                a,
+                b,
+                c
+            };
+            set.ExceptWith(new UInt256[] { a });
+            CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { b, c });
+
+            set = new FIFOSet<UInt256>(10)
+            {
+                a,
+                b,
+                c
+            };
+            set.ExceptWith(new UInt256[] { c });
+            CollectionAssert.AreEqual(set.ToArray(), new UInt256[] { a, b });
         }
     }
 }
diff --git a/neo/Helper.cs b/neo/Helper.cs
index 1a513ec79e..fa3b384a35 100644
--- a/neo/Helper.cs
+++ b/neo/Helper.cs
@@ -1,5 +1,5 @@
 using Microsoft.Extensions.Configuration;
-using Neo.Plugins;
+using Neo.IO.Caching;
 using System;
 using System.Collections.Generic;
 using System.Globalization;
@@ -54,6 +54,43 @@ internal static int GetLowestSetBit(this BigInteger i)
             throw new Exception();
         }
 
+        internal static void Remove<T>(this HashSet<T> set, ISet<T> other)
+        {
+            if (set.Count > other.Count)
+            {
+                set.ExceptWith(other);
+            }
+            else
+            {
+                set.RemoveWhere(u => other.Contains(u));
+            }
+        }
+
+        internal static void Remove<T>(this HashSet<T> set, FIFOSet<T> other)
+            where T : IEquatable<T>
+        {
+            if (set.Count > other.Count)
+            {
+                set.ExceptWith(other);
+            }
+            else
+            {
+                set.RemoveWhere(u => other.Contains(u));
+            }
+        }
+
+        internal static void Remove<T, V>(this HashSet<T> set, IReadOnlyDictionary<T, V> other)
+        {
+            if (set.Count > other.Count)
+            {
+                set.ExceptWith(other.Keys);
+            }
+            else
+            {
+                set.RemoveWhere(u => other.ContainsKey(u));
+            }
+        }
+
         internal static string GetVersion(this Assembly assembly)
         {
             CustomAttributeData attribute = assembly.CustomAttributes.FirstOrDefault(p => p.AttributeType == typeof(AssemblyInformationalVersionAttribute));
diff --git a/neo/IO/Caching/FIFOSet.cs b/neo/IO/Caching/FIFOSet.cs
index 98733ce444..af65db1b8b 100644
--- a/neo/IO/Caching/FIFOSet.cs
+++ b/neo/IO/Caching/FIFOSet.cs
@@ -6,12 +6,14 @@
 
 namespace Neo.IO.Caching
 {
-    internal class FIFOSet<T> : IEnumerable<T> where T : IEquatable<T>
+    internal class FIFOSet<T> : IReadOnlyCollection<T> where T : IEquatable<T>
     {
         private readonly int maxCapacity;
         private readonly int removeCount;
         private readonly OrderedDictionary dictionary;
 
+        public int Count => dictionary.Count;
+
         public FIFOSet(int maxCapacity, decimal batchSize = 0.1m)
         {
             if (maxCapacity <= 0) throw new ArgumentOutOfRangeException(nameof(maxCapacity));
@@ -46,11 +48,11 @@ public bool Contains(T item)
             return dictionary.Contains(item);
         }
 
-        public void ExceptWith(IEnumerable<UInt256> hashes)
+        public void ExceptWith(IEnumerable<T> entries)
         {
-            foreach (var hash in hashes)
+            foreach (var entry in entries)
             {
-                dictionary.Remove(hash);
+                dictionary.Remove(entry);
             }
         }
 
diff --git a/neo/Network/P2P/TaskManager.cs b/neo/Network/P2P/TaskManager.cs
index 2c0f7ae07b..82e2e9b87a 100644
--- a/neo/Network/P2P/TaskManager.cs
+++ b/neo/Network/P2P/TaskManager.cs
@@ -58,11 +58,11 @@ private void OnNewTasks(InvPayload payload)
                 return;
             }
             HashSet<UInt256> hashes = new HashSet<UInt256>(payload.Hashes);
-            hashes.ExceptWith(knownHashes);
+            hashes.Remove(knownHashes);
             if (payload.Type == InventoryType.Block)
                 session.AvailableTasks.UnionWith(hashes.Where(p => globalTasks.ContainsKey(p)));
 
-            hashes.ExceptWith(globalTasks.Keys);
+            hashes.Remove(globalTasks);
             if (hashes.Count == 0)
             {
                 RequestTasks(session);
@@ -203,7 +203,7 @@ private void RequestTasks(TaskSession session)
             if (session.HasTask) return;
             if (session.AvailableTasks.Count > 0)
             {
-                session.AvailableTasks.ExceptWith(knownHashes);
+                session.AvailableTasks.Remove(knownHashes);
                 session.AvailableTasks.RemoveWhere(p => Blockchain.Singleton.ContainsBlock(p));
                 HashSet<UInt256> hashes = new HashSet<UInt256>(session.AvailableTasks);
                 if (hashes.Count > 0)
@@ -213,7 +213,7 @@ private void RequestTasks(TaskSession session)
                         if (!IncrementGlobalTask(hash))
                             hashes.Remove(hash);
                     }
-                    session.AvailableTasks.ExceptWith(hashes);
+                    session.AvailableTasks.Remove(hashes);
                     foreach (UInt256 hash in hashes)
                         session.Tasks[hash] = DateTime.UtcNow;
                     foreach (InvPayload group in InvPayload.CreateGroup(InventoryType.Block, hashes.ToArray()))
diff --git a/neo/UIntBase.cs b/neo/UIntBase.cs
index af26e36d0a..6b5f6d6acd 100644
--- a/neo/UIntBase.cs
+++ b/neo/UIntBase.cs
@@ -2,6 +2,7 @@
 using System;
 using System.IO;
 using System.Linq;
+using System.Runtime.CompilerServices;
 
 namespace Neo
 {
@@ -110,6 +111,7 @@ void ISerializable.Serialize(BinaryWriter writer)
         /// <summary>
         /// Method ToArray() returns the byte array data_bytes, which stores the little-endian unsigned int
         /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public byte[] ToArray()
         {
             return data_bytes;