From 3ef7b5ec82bc1c91694ac2ab47ac6a2658490759 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Tue, 1 Jan 2019 17:09:42 +0800 Subject: [PATCH 01/27] Handles escape characters in JSON --- neo/IO/Json/JString.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/neo/IO/Json/JString.cs b/neo/IO/Json/JString.cs index 3aec381fb4..c9fac5a9a2 100644 --- a/neo/IO/Json/JString.cs +++ b/neo/IO/Json/JString.cs @@ -89,10 +89,18 @@ internal static JString Parse(TextReader reader) if (c == '\\') { c = (char)reader.Read(); - if (c == 'u') + switch (c) { - reader.Read(buffer, 0, 4); - c = (char)short.Parse(new string(buffer), NumberStyles.HexNumber); + case 'u': + reader.Read(buffer, 0, 4); + c = (char)short.Parse(new string(buffer), NumberStyles.HexNumber); + break; + case 'r': + c = '\r'; + break; + case 'n': + c = '\n'; + break; } } sb.Append(c); From c0f15005fda23030cd272831c4a8363a9aad4bcf Mon Sep 17 00:00:00 2001 From: Sng Ping Chiang Date: Tue, 1 Jan 2019 22:30:54 +0800 Subject: [PATCH 02/27] Pass ApplicationExecution to IPersistencePlugin (#531) --- neo/Ledger/Blockchain.cs | 11 ++++++++--- neo/Plugins/IPersistencePlugin.cs | 4 +++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/neo/Ledger/Blockchain.cs b/neo/Ledger/Blockchain.cs index 73f123c5af..94a7ec2274 100644 --- a/neo/Ledger/Blockchain.cs +++ b/neo/Ledger/Blockchain.cs @@ -455,6 +455,7 @@ private void Persist(Block block) { using (Snapshot snapshot = GetSnapshot()) { + List all_application_executed = new List(); snapshot.PersistingBlock = block; snapshot.Blocks.Add(block.Hash, new BlockState { @@ -605,11 +606,15 @@ private void Persist(Block block) break; } if (execution_results.Count > 0) - Distribute(new ApplicationExecuted + { + ApplicationExecuted application_executed = new ApplicationExecuted { Transaction = tx, ExecutionResults = execution_results.ToArray() - }); + }; + Distribute(application_executed); + all_application_executed.Add(application_executed); + } } snapshot.BlockHashIndex.GetAndChange().Hash = block.Hash; snapshot.BlockHashIndex.GetAndChange().Index = block.Index; @@ -620,7 +625,7 @@ private void Persist(Block block) snapshot.HeaderHashIndex.GetAndChange().Index = block.Index; } foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins) - plugin.OnPersist(snapshot); + plugin.OnPersist(snapshot, all_application_executed); snapshot.Commit(); } UpdateCurrentSnapshot(); diff --git a/neo/Plugins/IPersistencePlugin.cs b/neo/Plugins/IPersistencePlugin.cs index d70afc53fa..cf4ee697fd 100644 --- a/neo/Plugins/IPersistencePlugin.cs +++ b/neo/Plugins/IPersistencePlugin.cs @@ -1,9 +1,11 @@ using Neo.Persistence; +using System.Collections.Generic; +using static Neo.Ledger.Blockchain; namespace Neo.Plugins { public interface IPersistencePlugin { - void OnPersist(Snapshot snapshot); + void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList); } } From 338ab7adff7f38f47f8bb63950d885d3e06e0d15 Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Wed, 2 Jan 2019 16:32:39 +0800 Subject: [PATCH 03/27] Update dependencies: (#532) - Akka 1.3.11 - Microsoft.AspNetCore.ResponseCompression 2.2.0 - Microsoft.AspNetCore.Server.Kestrel 2.2.0 - Microsoft.AspNetCore.Server.Kestrel.Https 2.2.0 - Microsoft.AspNetCore.WebSockets 2.2.0 - Microsoft.EntityFrameworkCore.Sqlite 2.2.0 - Microsoft.Extensions.Configuration.Json 2.2.0 --- neo/neo.csproj | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/neo/neo.csproj b/neo/neo.csproj index e322842ded..f15a85423b 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -32,13 +32,13 @@ - - - - - - - + + + + + + + From 40ec6b2f364b809e3a91d706b27931f5b4468fe1 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 2 Jan 2019 16:36:44 +0800 Subject: [PATCH 04/27] change version to v2.9.4 --- neo/neo.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/neo.csproj b/neo/neo.csproj index f15a85423b..d0aa64c390 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -3,7 +3,7 @@ 2015-2018 The Neo Project Neo - 2.9.3 + 2.9.4 The Neo Project netstandard2.0;net47 true From c16475e7c11c6e1fcf81a062a825ee5e30089daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Mon, 7 Jan 2019 16:50:36 -0200 Subject: [PATCH 05/27] Updating Unknown to Policy Fail (#533) --- neo/Ledger/Blockchain.cs | 2 +- neo/Ledger/RelayResultReason.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/Ledger/Blockchain.cs b/neo/Ledger/Blockchain.cs index 94a7ec2274..db9ecb1cf5 100644 --- a/neo/Ledger/Blockchain.cs +++ b/neo/Ledger/Blockchain.cs @@ -388,7 +388,7 @@ private RelayResultReason OnNewTransaction(Transaction transaction) if (!transaction.Verify(currentSnapshot, GetMemoryPool())) return RelayResultReason.Invalid; if (!Plugin.CheckPolicy(transaction)) - return RelayResultReason.Unknown; + return RelayResultReason.PolicyFail; if (!mem_pool.TryAdd(transaction.Hash, transaction)) return RelayResultReason.OutOfMemory; diff --git a/neo/Ledger/RelayResultReason.cs b/neo/Ledger/RelayResultReason.cs index f66504846f..e698d0ea34 100644 --- a/neo/Ledger/RelayResultReason.cs +++ b/neo/Ledger/RelayResultReason.cs @@ -7,6 +7,7 @@ public enum RelayResultReason : byte OutOfMemory, UnableToVerify, Invalid, + PolicyFail, Unknown } } From b1311ba44db30b9839944785310509ddb160aa98 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 9 Jan 2019 13:07:16 +0800 Subject: [PATCH 06/27] Fix a dead lock in `WalletIndexer` --- neo/Wallets/WalletIndexer.cs | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/neo/Wallets/WalletIndexer.cs b/neo/Wallets/WalletIndexer.cs index 5e1b347ed7..3581dd6728 100644 --- a/neo/Wallets/WalletIndexer.cs +++ b/neo/Wallets/WalletIndexer.cs @@ -127,8 +127,9 @@ public IEnumerable GetTransactions(IEnumerable accounts) yield return hash; } - private void ProcessBlock(Block block, HashSet accounts, WriteBatch batch) + private (Transaction, UInt160[])[] ProcessBlock(Block block, HashSet accounts, WriteBatch batch) { + var change_set = new List<(Transaction, UInt160[])>(); foreach (Transaction tx in block.Transactions) { HashSet accounts_changed = new HashSet(); @@ -218,15 +219,10 @@ private void ProcessBlock(Block block, HashSet accounts, WriteBatch bat { foreach (UInt160 account in accounts_changed) batch.Put(SliceBuilder.Begin(DataEntryPrefix.ST_Transaction).Add(account).Add(tx.Hash), false); - WalletTransaction?.Invoke(null, new WalletTransactionEventArgs - { - Transaction = tx, - RelatedAccounts = accounts_changed.ToArray(), - Height = block.Index, - Time = block.Timestamp - }); + change_set.Add((tx, accounts_changed.ToArray())); } } + return change_set.ToArray(); } private void ProcessBlocks() @@ -234,15 +230,18 @@ private void ProcessBlocks() while (!disposed) { while (!disposed) + { + Block block; + (Transaction, UInt160[])[] change_set; lock (SyncRoot) { if (indexes.Count == 0) break; uint height = indexes.Keys.Min(); - Block block = Blockchain.Singleton.Store.GetBlock(height); + block = Blockchain.Singleton.Store.GetBlock(height); if (block == null) break; WriteBatch batch = new WriteBatch(); HashSet accounts = indexes[height]; - ProcessBlock(block, accounts, batch); + change_set = ProcessBlock(block, accounts, batch); ReadOptions options = ReadOptions.Default; byte[] groupId = db.Get(options, SliceBuilder.Begin(DataEntryPrefix.IX_Group).Add(height)).ToArray(); indexes.Remove(height); @@ -261,6 +260,17 @@ private void ProcessBlocks() } db.Write(WriteOptions.Default, batch); } + foreach (var (tx, accounts) in change_set) + { + WalletTransaction?.Invoke(null, new WalletTransactionEventArgs + { + Transaction = tx, + RelatedAccounts = accounts, + Height = block.Index, + Time = block.Timestamp + }); + } + } for (int i = 0; i < 20 && !disposed; i++) Thread.Sleep(100); } From 8cecbbc8f9f0a2e15346ce7d7e3b9e56e6bf257d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Wed, 9 Jan 2019 10:45:56 -0200 Subject: [PATCH 07/27] Downgrade Sqlite to 2.1.4 (#535) --- neo/neo.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/neo.csproj b/neo/neo.csproj index d0aa64c390..f922aae876 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -37,7 +37,7 @@ - + From b9e38fca02c91533772bb3b01ad2e70403469d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Fri, 11 Jan 2019 09:24:40 -0200 Subject: [PATCH 08/27] RPC call gettransactionheight (#541) * getrawtransactionheight Nowadays two calls are need to get a transaction height, `getrawtransaction` with `verbose` and then use the `blockhash`. Other option is to use `confirmations`, but it can be misleading. * Minnor fix * Shargon's tip * modified --- neo/Network/RPC/RpcServer.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index 5bf4b21f8d..f02cbbe922 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -331,6 +331,13 @@ private JObject Process(string method, JArray _params) }) ?? new StorageItem(); return item.Value?.ToHexString(); } + case "gettransactionheight": + { + UInt256 hash = UInt256.Parse(_params[0].AsString()); + uint? height = Blockchain.Singleton.Store.GetTransactions().TryGet(hash)?.BlockIndex; + if (height.HasValue) return height.Value; + throw new RpcException(-100, "Unknown transaction"); + } case "gettxout": { UInt256 hash = UInt256.Parse(_params[0].AsString()); From 74882737439fea2a874a25acacd17863044adfc6 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 11 Jan 2019 16:59:22 +0100 Subject: [PATCH 09/27] Allow to use the wallet inside a RPC plugin (#536) --- neo/Network/RPC/RpcServer.cs | 85 ++++++++++++++++++------------------ 1 file changed, 43 insertions(+), 42 deletions(-) diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index f02cbbe922..0c090c770f 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -1,4 +1,14 @@ -using Akka.Actor; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; +using Akka.Actor; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; @@ -16,30 +26,21 @@ using Neo.VM; using Neo.Wallets; using Neo.Wallets.NEP6; -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Net.Security; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; namespace Neo.Network.RPC { public sealed class RpcServer : IDisposable { - private readonly NeoSystem system; - private Wallet wallet; + public Wallet Wallet; + private IWebHost host; private Fixed8 maxGasInvoke; + private readonly NeoSystem system; public RpcServer(NeoSystem system, Wallet wallet = null, Fixed8 maxGasInvoke = default(Fixed8)) { this.system = system; - this.wallet = wallet; + this.Wallet = wallet; this.maxGasInvoke = maxGasInvoke; } @@ -86,7 +87,7 @@ private JObject GetInvokeResult(byte[] script) { json["stack"] = "error: recursive reference"; } - if (wallet != null) + if (Wallet != null) { InvocationTransaction tx = new InvocationTransaction { @@ -97,11 +98,11 @@ private JObject GetInvokeResult(byte[] script) tx.Gas -= Fixed8.FromDecimal(10); if (tx.Gas < Fixed8.Zero) tx.Gas = Fixed8.Zero; tx.Gas = tx.Gas.Ceiling(); - tx = wallet.MakeTransaction(tx); + tx = Wallet.MakeTransaction(tx); if (tx != null) { ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) tx.Witnesses = context.GetWitnesses(); else @@ -133,7 +134,7 @@ private static JObject GetRelayResult(RelayResultReason reason) public void OpenWallet(Wallet wallet) { - this.wallet = wallet; + this.Wallet = wallet; } private JObject Process(string method, JArray _params) @@ -141,12 +142,12 @@ private JObject Process(string method, JArray _params) switch (method) { case "dumpprivkey": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied"); else { UInt160 scriptHash = _params[0].AsString().ToScriptHash(); - WalletAccount account = wallet.GetAccount(scriptHash); + WalletAccount account = Wallet.GetAccount(scriptHash); return account.GetKey().Export(); } case "getaccountstate": @@ -162,7 +163,7 @@ private JObject Process(string method, JArray _params) return asset?.ToJson() ?? throw new RpcException(-100, "Unknown asset"); } case "getbalance": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied."); else { @@ -170,10 +171,10 @@ private JObject Process(string method, JArray _params) switch (UIntBase.Parse(_params[0].AsString())) { case UInt160 asset_id_160: //NEP-5 balance - json["balance"] = wallet.GetAvailable(asset_id_160).ToString(); + json["balance"] = Wallet.GetAvailable(asset_id_160).ToString(); break; case UInt256 asset_id_256: //Global Assets balance - IEnumerable coins = wallet.GetCoins().Where(p => !p.State.HasFlag(CoinState.Spent) && p.Output.AssetId.Equals(asset_id_256)); + IEnumerable coins = Wallet.GetCoins().Where(p => !p.State.HasFlag(CoinState.Spent) && p.Output.AssetId.Equals(asset_id_256)); json["balance"] = coins.Sum(p => p.Output.Value).ToString(); json["confirmed"] = coins.Where(p => p.State.HasFlag(CoinState.Confirmed)).Sum(p => p.Output.Value).ToString(); break; @@ -267,12 +268,12 @@ private JObject Process(string method, JArray _params) return contract?.ToJson() ?? throw new RpcException(-100, "Unknown contract"); } case "getnewaddress": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied"); else { - WalletAccount account = wallet.CreateAccount(); - if (wallet is NEP6Wallet nep6) + WalletAccount account = Wallet.CreateAccount(); + if (Wallet is NEP6Wallet nep6) nep6.Save(); return account.Address; } @@ -366,10 +367,10 @@ private JObject Process(string method, JArray _params) return json; } case "getwalletheight": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied."); else - return (wallet.WalletHeight > 0) ? wallet.WalletHeight - 1 : 0; + return (Wallet.WalletHeight > 0) ? Wallet.WalletHeight - 1 : 0; case "invoke": { UInt160 script_hash = UInt160.Parse(_params[0].AsString()); @@ -399,10 +400,10 @@ private JObject Process(string method, JArray _params) return GetInvokeResult(script); } case "listaddress": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied."); else - return wallet.GetAccounts().Select(p => + return Wallet.GetAccounts().Select(p => { JObject account = new JObject(); account["address"] = p.Address; @@ -412,7 +413,7 @@ private JObject Process(string method, JArray _params) return account; }).ToArray(); case "sendfrom": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied"); else { @@ -427,7 +428,7 @@ private JObject Process(string method, JArray _params) if (fee < Fixed8.Zero) throw new RpcException(-32602, "Invalid params"); UInt160 change_address = _params.Count >= 6 ? _params[5].AsString().ToScriptHash() : null; - Transaction tx = wallet.MakeTransaction(null, new[] + Transaction tx = Wallet.MakeTransaction(null, new[] { new TransferOutput { @@ -439,11 +440,11 @@ private JObject Process(string method, JArray _params) if (tx == null) throw new RpcException(-300, "Insufficient funds"); ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) { tx.Witnesses = context.GetWitnesses(); - wallet.ApplyTransaction(tx); + Wallet.ApplyTransaction(tx); system.LocalNode.Tell(new LocalNode.Relay { Inventory = tx }); return tx.ToJson(); } @@ -453,7 +454,7 @@ private JObject Process(string method, JArray _params) } } case "sendmany": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied"); else { @@ -478,15 +479,15 @@ private JObject Process(string method, JArray _params) if (fee < Fixed8.Zero) throw new RpcException(-32602, "Invalid params"); UInt160 change_address = _params.Count >= 3 ? _params[2].AsString().ToScriptHash() : null; - Transaction tx = wallet.MakeTransaction(null, outputs, change_address: change_address, fee: fee); + Transaction tx = Wallet.MakeTransaction(null, outputs, change_address: change_address, fee: fee); if (tx == null) throw new RpcException(-300, "Insufficient funds"); ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) { tx.Witnesses = context.GetWitnesses(); - wallet.ApplyTransaction(tx); + Wallet.ApplyTransaction(tx); system.LocalNode.Tell(new LocalNode.Relay { Inventory = tx }); return tx.ToJson(); } @@ -502,7 +503,7 @@ private JObject Process(string method, JArray _params) return GetRelayResult(reason); } case "sendtoaddress": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied"); else { @@ -516,7 +517,7 @@ private JObject Process(string method, JArray _params) if (fee < Fixed8.Zero) throw new RpcException(-32602, "Invalid params"); UInt160 change_address = _params.Count >= 5 ? _params[4].AsString().ToScriptHash() : null; - Transaction tx = wallet.MakeTransaction(null, new[] + Transaction tx = Wallet.MakeTransaction(null, new[] { new TransferOutput { @@ -528,11 +529,11 @@ private JObject Process(string method, JArray _params) if (tx == null) throw new RpcException(-300, "Insufficient funds"); ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) { tx.Witnesses = context.GetWitnesses(); - wallet.ApplyTransaction(tx); + Wallet.ApplyTransaction(tx); system.LocalNode.Tell(new LocalNode.Relay { Inventory = tx }); return tx.ToJson(); } From 5222283c9f718217d550ca789a32c383d17749d3 Mon Sep 17 00:00:00 2001 From: jsolman Date: Sat, 12 Jan 2019 22:37:36 -0800 Subject: [PATCH 10/27] Improve Large MemoryPool Performance - Sort + intelligent TX reverification (#500) Improve Large MemoryPool Performance - Sort + intelligent TX reverification (#500) * Keep both verified and unverified (previously verified) transactions in sorted trees so ejecting transactions above the pool size is a low latency operation. * Re-verify unverified transactions when Blockchain actor is idle. * Don't re-verify transactions needlessly when not at the tip of the chain. * Support passing a flag to `getrawmempool` to retrieve both verified and unverified TX hashes. * Support MaxTransactionsPerBlock and MaxFreeTransactionsPerBlock from Policy plugins. * Rebroadcast re-verified transactions if it has been a while since last broadcast (high priority transactions are rebroadcast more frequently than low priority transactions. --- neo/Consensus/ConsensusContext.cs | 2 +- neo/Consensus/ConsensusService.cs | 9 +- neo/Ledger/Blockchain.cs | 53 +-- neo/Ledger/MemoryPool.cs | 506 ++++++++++++++++++++++++----- neo/Network/P2P/ProtocolHandler.cs | 2 +- neo/Network/RPC/RpcServer.cs | 15 +- neo/Plugins/IPolicyPlugin.cs | 2 + 7 files changed, 468 insertions(+), 121 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index d7d9b1bcfd..924709fc9e 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -202,7 +202,7 @@ public void Reset() public void Fill() { - IEnumerable mem_pool = Blockchain.Singleton.GetMemoryPool(); + IEnumerable mem_pool = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); foreach (IPolicyPlugin plugin in Plugin.Policies) mem_pool = plugin.FilterForBlock(mem_pool); List transactions = mem_pool.ToList(); diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 77b727e0c8..6cd80b38f9 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -219,19 +219,20 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m if (!Crypto.Default.VerifySignature(hashData, context.Signatures[i], context.Validators[i].EncodePoint(false))) context.Signatures[i] = null; context.Signatures[payload.ValidatorIndex] = message.Signature; - Dictionary mempool = Blockchain.Singleton.GetMemoryPool().ToDictionary(p => p.Hash); + Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); + List unverified = new List(); foreach (UInt256 hash in context.TransactionHashes.Skip(1)) { - if (mempool.TryGetValue(hash, out Transaction tx)) + if (mempoolVerified.TryGetValue(hash, out Transaction tx)) { if (!AddTransaction(tx, false)) return; } else { - tx = Blockchain.Singleton.GetUnverifiedTransaction(hash); - if (tx != null) + + if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) unverified.Add(tx); } } diff --git a/neo/Ledger/Blockchain.cs b/neo/Ledger/Blockchain.cs index db9ecb1cf5..fa9d7dc5fb 100644 --- a/neo/Ledger/Blockchain.cs +++ b/neo/Ledger/Blockchain.cs @@ -1,6 +1,5 @@ using Akka.Actor; using Akka.Configuration; -using Neo.Cryptography; using Neo.Cryptography.ECC; using Neo.IO; using Neo.IO.Actors; @@ -12,10 +11,8 @@ using Neo.SmartContract; using Neo.VM; using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Numerics; using System.Threading; namespace Neo.Ledger @@ -114,19 +111,20 @@ public class ImportCompleted { } } }; + private const int MemoryPoolMaxTransactions = 50_000; + private const int MaxTxToReverifyPerIdle = 10; private static readonly object lockObj = new object(); private readonly NeoSystem system; private readonly List header_index = new List(); private uint stored_header_count = 0; private readonly Dictionary block_cache = new Dictionary(); private readonly Dictionary> block_cache_unverified = new Dictionary>(); - private readonly MemoryPool mem_pool = new MemoryPool(50_000); - private readonly ConcurrentDictionary mem_pool_unverified = new ConcurrentDictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); private readonly HashSet subscribers = new HashSet(); private Snapshot currentSnapshot; public Store Store { get; } + public MemoryPool MemPool { get; } public uint Height => currentSnapshot.Height; public uint HeaderHeight => (uint)header_index.Count - 1; public UInt256 CurrentBlockHash => currentSnapshot.CurrentBlockHash; @@ -150,6 +148,7 @@ static Blockchain() public Blockchain(NeoSystem system, Store store) { this.system = system; + this.MemPool = new MemoryPool(system, MemoryPoolMaxTransactions); this.Store = store; lock (lockObj) { @@ -190,7 +189,7 @@ public bool ContainsBlock(UInt256 hash) public bool ContainsTransaction(UInt256 hash) { - if (mem_pool.ContainsKey(hash)) return true; + if (MemPool.ContainsKey(hash)) return true; return Store.ContainsTransaction(hash); } @@ -218,11 +217,6 @@ public static UInt160 GetConsensusAddress(ECPoint[] validators) return Contract.CreateMultiSigRedeemScript(validators.Length - (validators.Length - 1) / 3, validators).ToScriptHash(); } - public IEnumerable GetMemoryPool() - { - return mem_pool; - } - public Snapshot GetSnapshot() { return Store.GetSnapshot(); @@ -230,17 +224,11 @@ public Snapshot GetSnapshot() public Transaction GetTransaction(UInt256 hash) { - if (mem_pool.TryGetValue(hash, out Transaction transaction)) + if (MemPool.TryGetValue(hash, out Transaction transaction)) return transaction; return Store.GetTransaction(hash); } - internal Transaction GetUnverifiedTransaction(UInt256 hash) - { - mem_pool_unverified.TryGetValue(hash, out Transaction transaction); - return transaction; - } - private void OnImport(IEnumerable blocks) { foreach (Block block in blocks) @@ -264,7 +252,7 @@ private void AddUnverifiedBlockToCache(Block block) blocks.AddLast(block); } - + private RelayResultReason OnNewBlock(Block block) { if (block.Index <= Height) @@ -289,7 +277,7 @@ private RelayResultReason OnNewBlock(Block block) if (block.Index == Height + 1) { Block block_persist = block; - List blocksToPersistList = new List(); + List blocksToPersistList = new List(); while (true) { blocksToPersistList.Add(block_persist); @@ -315,7 +303,7 @@ private RelayResultReason OnNewBlock(Block block) if (block_cache_unverified.TryGetValue(Height + 1, out LinkedList unverifiedBlocks)) { foreach (var unverifiedBlock in unverifiedBlocks) - Self.Tell(unverifiedBlock, ActorRefs.NoSender); + Self.Tell(unverifiedBlock, ActorRefs.NoSender); block_cache_unverified.Remove(Height + 1); } } @@ -385,12 +373,14 @@ private RelayResultReason OnNewTransaction(Transaction transaction) return RelayResultReason.Invalid; if (ContainsTransaction(transaction.Hash)) return RelayResultReason.AlreadyExists; - if (!transaction.Verify(currentSnapshot, GetMemoryPool())) + if (!MemPool.CanTransactionFitInPool(transaction)) + return RelayResultReason.OutOfMemory; + if (!transaction.Verify(currentSnapshot, MemPool.GetVerifiedTransactions())) return RelayResultReason.Invalid; if (!Plugin.CheckPolicy(transaction)) return RelayResultReason.PolicyFail; - if (!mem_pool.TryAdd(transaction.Hash, transaction)) + if (!MemPool.TryAdd(transaction.Hash, transaction)) return RelayResultReason.OutOfMemory; system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = transaction }); @@ -400,18 +390,7 @@ private RelayResultReason OnNewTransaction(Transaction transaction) private void OnPersistCompleted(Block block) { block_cache.Remove(block.Hash); - foreach (Transaction tx in block.Transactions) - mem_pool.TryRemove(tx.Hash, out _); - mem_pool_unverified.Clear(); - foreach (Transaction tx in mem_pool - .OrderByDescending(p => p.NetworkFee / p.Size) - .ThenByDescending(p => p.NetworkFee) - .ThenByDescending(p => new BigInteger(p.Hash.ToArray()))) - { - mem_pool_unverified.TryAdd(tx.Hash, tx); - Self.Tell(tx, ActorRefs.NoSender); - } - mem_pool.Clear(); + MemPool.UpdatePoolForBlockPersisted(block, currentSnapshot); PersistCompleted completed = new PersistCompleted { Block = block }; system.Consensus?.Tell(completed); Distribute(completed); @@ -439,6 +418,10 @@ protected override void OnReceive(object message) case ConsensusPayload payload: Sender.Tell(OnNewConsensus(payload)); break; + case Idle _: + if (MemPool.ReVerifyTopUnverifiedTransactionsIfNeeded(MaxTxToReverifyPerIdle, currentSnapshot)) + Self.Tell(Idle.Instance, ActorRefs.NoSender); + break; case Terminated terminated: subscribers.Remove(terminated.ActorRef); break; diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 28aea62113..1d05679934 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -1,165 +1,513 @@ using Neo.Network.P2P.Payloads; using System; using System.Collections; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; +using Akka.Util.Internal; +using Neo.Network.P2P; +using Neo.Persistence; +using Neo.Plugins; namespace Neo.Ledger { - internal class MemoryPool : IReadOnlyCollection + public class MemoryPool : IReadOnlyCollection { - private class PoolItem + private class PoolItem : IComparable { public readonly Transaction Transaction; public readonly DateTime Timestamp; + public DateTime LastBroadcastTimestamp; public PoolItem(Transaction tx) { Transaction = tx; Timestamp = DateTime.UtcNow; + LastBroadcastTimestamp = Timestamp; + } + + public int CompareTo(Transaction tx) + { + if (tx == null) return 1; + int ret = Transaction.FeePerByte.CompareTo(tx.FeePerByte); + if (ret != 0) return ret; + ret = Transaction.NetworkFee.CompareTo(tx.NetworkFee); + if (ret != 0) return ret; + + return Transaction.Hash.CompareTo(tx.Hash); + } + + public int CompareTo(PoolItem otherItem) + { + if (otherItem == null) return 1; + return CompareTo(otherItem.Transaction); } } - private readonly ConcurrentDictionary _mem_pool_fee = new ConcurrentDictionary(); - private readonly ConcurrentDictionary _mem_pool_free = new ConcurrentDictionary(); + // Allow reverified transactions to be rebroadcast if it has been this many block times since last broadcast. + private const int BlocksTillRebroadcastLowPriorityPoolTx = 30; + private const int BlocksTillRebroadcastHighPriorityPoolTx = 10; + + private static readonly double MaxSecondsToReverifyHighPrioTx = (double) Blockchain.SecondsPerBlock / 3; + private static readonly double MaxSecondsToReverifyLowPrioTx = (double) Blockchain.SecondsPerBlock / 5; + + // These two are not expected to be hit, they are just safegaurds. + private static readonly double MaxSecondsToReverifyHighPrioTxPerIdle = (double) Blockchain.SecondsPerBlock / 15; + private static readonly double MaxSecondsToReverifyLowPrioTxPerIdle = (double) Blockchain.SecondsPerBlock / 30; + private readonly NeoSystem _system; + + // + /// + /// Guarantees consistency of the pool data structures. + /// + /// Note: The data structures are only modified from the `Blockchain` actor; so operations guaranteed to be + /// performed by the blockchain actor do not need to acquire the read lock; they only need the write + /// lock for write operations. + /// + private readonly ReaderWriterLockSlim _txRwLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + + /// + /// Store all verified unsorted transactions currently in the pool. + /// + private readonly Dictionary _unsortedTransactions = new Dictionary(); + /// + /// Stores the verified low priority sorted transactions currently in the pool. + /// + private readonly SortedSet _sortedLowPrioTransactions = new SortedSet(); + /// + /// Stores the verified high priority sorted transactins currently in the pool. + /// + private readonly SortedSet _sortedHighPrioTransactions = new SortedSet(); + + /// + /// Store the unverified transactions currently in the pool. + /// + /// Transactions in this data structure were valid in some prior block, but may no longer be valid. + /// The top ones that could make it into the next block get verified and moved into the verified data structures + /// (_unsortedTransactions, _sortedLowPrioTransactions, and _sortedHighPrioTransactions) after each block. + /// + private readonly Dictionary _unverifiedTransactions = new Dictionary(); + private readonly SortedSet _unverifiedSortedHighPriorityTransactions = new SortedSet(); + private readonly SortedSet _unverifiedSortedLowPriorityTransactions = new SortedSet(); + + private int _maxTxPerBlock; + private int _maxLowPriorityTxPerBlock; + + /// + /// Total maximum capacity of transactions the pool can hold. + /// public int Capacity { get; } - public int Count => _mem_pool_fee.Count + _mem_pool_free.Count; - public MemoryPool(int capacity) + /// + /// Total count of transactions in the pool. + /// + public int Count { + get + { + _txRwLock.EnterReadLock(); + try + { + return _unsortedTransactions.Count + _unverifiedTransactions.Count; + } + finally + { + _txRwLock.ExitReadLock(); + } + } + } + + /// + /// Total count of verified transactions in the pool. + /// + public int VerifiedCount => _unsortedTransactions.Count; // read of 32 bit type is atomic (no lock) + + public int UnVerifiedCount => _unverifiedTransactions.Count; + + public MemoryPool(NeoSystem system, int capacity) + { + _system = system; Capacity = capacity; + LoadMaxTxLimitsFromPolicyPlugins(); } - public void Clear() + public void LoadMaxTxLimitsFromPolicyPlugins() { - _mem_pool_free.Clear(); - _mem_pool_fee.Clear(); + _maxTxPerBlock = int.MaxValue; + _maxLowPriorityTxPerBlock = int.MaxValue; + foreach (IPolicyPlugin plugin in Plugin.Policies) + { + _maxTxPerBlock = Math.Min(_maxTxPerBlock, plugin.MaxTxPerBlock); + _maxLowPriorityTxPerBlock = Math.Min(_maxLowPriorityTxPerBlock, plugin.MaxLowPriorityTxPerBlock); + } + } + + /// + /// Determine whether the pool is holding this transaction and has at some point verified it. + /// Note: The pool may not have verified it since the last block was persisted. To get only the + /// transactions that have been verified during this block use GetVerifiedTransactions() + /// + /// the transaction hash + /// true if the MemoryPool contain the transaction + public bool ContainsKey(UInt256 hash) + { + _txRwLock.EnterReadLock(); + try + { + return _unsortedTransactions.ContainsKey(hash) + || _unverifiedTransactions.ContainsKey(hash); + } + finally + { + _txRwLock.ExitReadLock(); + } } - public bool ContainsKey(UInt256 hash) => _mem_pool_free.ContainsKey(hash) || _mem_pool_fee.ContainsKey(hash); + public bool TryGetValue(UInt256 hash, out Transaction tx) + { + _txRwLock.EnterReadLock(); + try + { + bool ret = _unsortedTransactions.TryGetValue(hash, out PoolItem item) + || _unverifiedTransactions.TryGetValue(hash, out item); + tx = ret ? item.Transaction : null; + return ret; + } + finally + { + _txRwLock.ExitReadLock(); + } + } + // Note: This isn't used in Fill during consensus, fill uses GetSortedVerifiedTransactions() public IEnumerator GetEnumerator() { - return - _mem_pool_fee.Select(p => p.Value.Transaction) - .Concat(_mem_pool_free.Select(p => p.Value.Transaction)) - .GetEnumerator(); + _txRwLock.EnterReadLock(); + try + { + return _unsortedTransactions.Select(p => p.Value.Transaction) + .Concat(_unverifiedTransactions.Select(p => p.Value.Transaction)) + .ToList() + .GetEnumerator(); + } + finally + { + _txRwLock.ExitReadLock(); + } } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - static void RemoveLowestFee(ConcurrentDictionary pool, int count) + public IEnumerable GetVerifiedTransactions() { - if (count <= 0) return; - if (count >= pool.Count) + _txRwLock.EnterReadLock(); + try { - pool.Clear(); + return _unsortedTransactions.Select(p => p.Value.Transaction).ToArray(); } - else + finally { - UInt256[] delete = pool.AsParallel() - .OrderBy(p => p.Value.Transaction.NetworkFee / p.Value.Transaction.Size) - .ThenBy(p => p.Value.Transaction.NetworkFee) - .ThenBy(p => new BigInteger(p.Key.ToArray())) - .Take(count) - .Select(p => p.Key) - .ToArray(); + _txRwLock.ExitReadLock(); + } + } - foreach (UInt256 hash in delete) - { - pool.TryRemove(hash, out _); - } + public void GetVerifiedAndUnverifiedTransactions(out IEnumerable verifiedTransactions, + out IEnumerable unverifiedTransactions) + { + _txRwLock.EnterReadLock(); + try + { + verifiedTransactions = _sortedHighPrioTransactions.Select(p => p.Transaction) + .Concat(_sortedLowPrioTransactions.Select(p => p.Transaction)).ToArray(); + unverifiedTransactions = _unverifiedTransactions.Select(p => p.Value.Transaction).ToArray(); + } + finally + { + _txRwLock.ExitReadLock(); + } + } + + public IEnumerable GetSortedVerifiedTransactions() + { + _txRwLock.EnterReadLock(); + try + { + return _sortedHighPrioTransactions.Select(p => p.Transaction) + .Concat(_sortedLowPrioTransactions.Select(p => p.Transaction)) + .ToArray(); + } + finally + { + _txRwLock.ExitReadLock(); } } - static void RemoveOldest(ConcurrentDictionary pool, DateTime time) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private PoolItem GetLowestFeeTransaction(SortedSet verifiedTxSorted, + SortedSet unverifiedTxSorted, out SortedSet sortedPool) { - UInt256[] hashes = pool - .Where(p => p.Value.Timestamp < time) - .Select(p => p.Key) - .ToArray(); + PoolItem minItem = unverifiedTxSorted.Min; + sortedPool = minItem != null ? unverifiedTxSorted : null; + + PoolItem verifiedMin = verifiedTxSorted.Min; + if (verifiedMin == null) return minItem; + + if (minItem != null && verifiedMin.CompareTo(minItem) >= 0) + return minItem; - foreach (UInt256 hash in hashes) + sortedPool = verifiedTxSorted; + minItem = verifiedMin; + + return minItem; + } + + private PoolItem GetLowestFeeTransaction(out Dictionary unsortedTxPool, out SortedSet sortedPool) + { + var minItem = GetLowestFeeTransaction(_sortedLowPrioTransactions, _unverifiedSortedLowPriorityTransactions, + out sortedPool); + + if (minItem != null) { - pool.TryRemove(hash, out _); + unsortedTxPool = sortedPool == _unverifiedSortedLowPriorityTransactions + ? _unverifiedTransactions : _unsortedTransactions; + return minItem; + } + + try + { + return GetLowestFeeTransaction(_sortedHighPrioTransactions, _unverifiedSortedHighPriorityTransactions, + out sortedPool); + } + finally + { + unsortedTxPool = sortedPool == _unverifiedSortedHighPriorityTransactions + ? _unverifiedTransactions : _unsortedTransactions; } } - public bool TryAdd(UInt256 hash, Transaction tx) + // Note: this must only be called from a single thread (the Blockchain actor) + internal bool CanTransactionFitInPool(Transaction tx) { - ConcurrentDictionary pool; + if (Count < Capacity) return true; + + return GetLowestFeeTransaction(out _, out _).CompareTo(tx) <= 0; + } - if (tx.IsLowPriority) + /// + /// + /// Note: This must only be called from a single thread (the Blockchain actor) to add a transaction to the pool + /// one should tell the Blockchain actor about the transaction. + /// + /// + /// + /// + internal bool TryAdd(UInt256 hash, Transaction tx) + { + var poolItem = new PoolItem(tx); + + if (_unsortedTransactions.ContainsKey(hash)) return false; + + _txRwLock.EnterWriteLock(); + try { - pool = _mem_pool_free; + _unsortedTransactions.Add(hash, poolItem); + + SortedSet pool = tx.IsLowPriority ? _sortedLowPrioTransactions : _sortedHighPrioTransactions; + pool.Add(poolItem); + RemoveOverCapacity(); } - else + finally { - pool = _mem_pool_fee; + _txRwLock.ExitWriteLock(); } - pool.TryAdd(hash, new PoolItem(tx)); + return _unsortedTransactions.ContainsKey(hash); + } - if (Count > Capacity) + private void RemoveOverCapacity() + { + while (Count > Capacity) { - RemoveOldest(_mem_pool_free, DateTime.UtcNow.AddSeconds(-Blockchain.SecondsPerBlock * 20)); + PoolItem minItem = GetLowestFeeTransaction(out var unsortedPool, out var sortedPool); - var exceed = Count - Capacity; + unsortedPool.Remove(minItem.Transaction.Hash); + sortedPool.Remove(minItem); + } + } - if (exceed > 0) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryRemoveVerified(UInt256 hash, out PoolItem item) + { + if (!_unsortedTransactions.TryGetValue(hash, out item)) + return false; + + _unsortedTransactions.Remove(hash); + SortedSet pool = item.Transaction.IsLowPriority + ? _sortedLowPrioTransactions : _sortedHighPrioTransactions; + pool.Remove(item); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryRemoveUnVerified(UInt256 hash, out PoolItem item) + { + if (!_unverifiedTransactions.TryGetValue(hash, out item)) + return false; + + _unverifiedTransactions.Remove(hash); + SortedSet pool = item.Transaction.IsLowPriority + ? _unverifiedSortedLowPriorityTransactions : _unverifiedSortedHighPriorityTransactions; + pool.Remove(item); + return true; + } + + // Note: this must only be called from a single thread (the Blockchain actor) + internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) + { + _txRwLock.EnterWriteLock(); + try + { + // First remove the transactions verified in the block. + foreach (Transaction tx in block.Transactions) { - RemoveLowestFee(_mem_pool_free, exceed); - exceed = Count - Capacity; + if (TryRemoveVerified(tx.Hash, out _)) continue; + TryRemoveUnVerified(tx.Hash, out _); + } - if (exceed > 0) - { - RemoveLowestFee(_mem_pool_fee, exceed); - } + // Add all the previously verified transactions back to the unverified transactions + foreach (PoolItem item in _sortedHighPrioTransactions) + { + if (_unverifiedTransactions.TryAdd(item.Transaction.Hash, item)) + _unverifiedSortedHighPriorityTransactions.Add(item); } + + foreach (PoolItem item in _sortedLowPrioTransactions) + { + if (_unverifiedTransactions.TryAdd(item.Transaction.Hash, item)) + _unverifiedSortedLowPriorityTransactions.Add(item); + } + + // Clear the verified transactions now, since they all must be reverified. + _unsortedTransactions.Clear(); + _sortedHighPrioTransactions.Clear(); + _sortedLowPrioTransactions.Clear(); + } + finally + { + _txRwLock.ExitWriteLock(); } - return pool.ContainsKey(hash); + // If we know about headers of future blocks, no point in verifying transactions from the unverified tx pool + // until we get caught up. + if (block.Index < Blockchain.Singleton.HeaderHeight) + return; + + if (Plugin.Policies.Count == 0) + return; + + LoadMaxTxLimitsFromPolicyPlugins(); + + ReverifyTransactions(_sortedHighPrioTransactions, _unverifiedSortedHighPriorityTransactions, + _maxTxPerBlock, MaxSecondsToReverifyHighPrioTx, snapshot); + ReverifyTransactions(_sortedLowPrioTransactions, _unverifiedSortedLowPriorityTransactions, + _maxLowPriorityTxPerBlock, MaxSecondsToReverifyLowPrioTx, snapshot); } - public bool TryRemove(UInt256 hash, out Transaction tx) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int ReverifyTransactions(SortedSet verifiedSortedTxPool, + SortedSet unverifiedSortedTxPool, int count, double secondsTimeout, Snapshot snapshot) { - if (_mem_pool_free.TryRemove(hash, out PoolItem item)) + DateTime reverifyCutOffTimeStamp = DateTime.UtcNow.AddSeconds(secondsTimeout); + + List reverifiedItems = new List(count); + List invalidItems = new List(); + foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) { - tx = item.Transaction; - return true; + // Re-verify the top fee max high priority transactions that can be verified in a block + if (item.Transaction.Verify(snapshot, _unsortedTransactions.Select(p => p.Value.Transaction))) + reverifiedItems.Add(item); + else // Transaction no longer valid -- will be removed from unverifiedTxPool. + invalidItems.Add(item); + + if (DateTime.UtcNow > reverifyCutOffTimeStamp) break; } - else if (_mem_pool_fee.TryRemove(hash, out item)) + + _txRwLock.EnterWriteLock(); + try { - tx = item.Transaction; - return true; + int blocksTillRebroadcast = unverifiedSortedTxPool == _sortedHighPrioTransactions + ? BlocksTillRebroadcastHighPriorityPoolTx : BlocksTillRebroadcastLowPriorityPoolTx; + + var rebroadcastCutOffTime = DateTime.UtcNow.AddSeconds( + -Blockchain.SecondsPerBlock * blocksTillRebroadcast); + foreach (PoolItem item in reverifiedItems) + { + if (_unsortedTransactions.TryAdd(item.Transaction.Hash, item)) + { + verifiedSortedTxPool.Add(item); + + if (item.LastBroadcastTimestamp < rebroadcastCutOffTime) + { + _system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = item.Transaction }, _system.Blockchain); + item.LastBroadcastTimestamp = DateTime.UtcNow; + } + } + + _unverifiedTransactions.Remove(item.Transaction.Hash); + unverifiedSortedTxPool.Remove(item); + } + + foreach (PoolItem item in invalidItems) + { + _unverifiedTransactions.Remove(item.Transaction.Hash); + unverifiedSortedTxPool.Remove(item); + } } - else + finally { - tx = null; - return false; + _txRwLock.ExitWriteLock(); } + + return reverifiedItems.Count; } - public bool TryGetValue(UInt256 hash, out Transaction tx) + /// + /// Reverify up to a given maximum count of transactions. Verifies less at a time once the max that can be + /// persisted per block has been reached. + /// + /// Note: this must only be called from a single thread (the Blockchain actor) + /// + /// Max transactions to reverify, the value passed should be >=2. If 1 is passed it + /// will still potentially use 2. + /// The snapshot to use for verifying. + /// true if more unsorted messages exist, otherwise false + internal bool ReVerifyTopUnverifiedTransactionsIfNeeded(int maxToVerify, Snapshot snapshot) { - if (_mem_pool_free.TryGetValue(hash, out PoolItem item)) - { - tx = item.Transaction; - return true; - } - else if (_mem_pool_fee.TryGetValue(hash, out item)) + if (Blockchain.Singleton.Height < Blockchain.Singleton.HeaderHeight) + return false; + + if (_unverifiedSortedHighPriorityTransactions.Count > 0) { - tx = item.Transaction; - return true; + // Always leave at least 1 tx for low priority tx + int verifyCount = _sortedHighPrioTransactions.Count > _maxTxPerBlock || maxToVerify == 1 + ? 1 : maxToVerify - 1; + maxToVerify -= ReverifyTransactions(_sortedHighPrioTransactions, _unverifiedSortedHighPriorityTransactions, + verifyCount, MaxSecondsToReverifyHighPrioTxPerIdle, snapshot); + + if (maxToVerify == 0) maxToVerify++; } - else + + if (_unverifiedSortedLowPriorityTransactions.Count > 0) { - tx = null; - return false; + int verifyCount = _sortedLowPrioTransactions.Count > _maxLowPriorityTxPerBlock + ? 1 : maxToVerify; + ReverifyTransactions(_sortedLowPrioTransactions, _unverifiedSortedLowPriorityTransactions, + verifyCount, MaxSecondsToReverifyLowPrioTxPerIdle, snapshot); } + + return _unverifiedTransactions.Count > 0; } } } diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 572c9b92fe..94766528de 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -264,7 +264,7 @@ private void OnInvMessageReceived(InvPayload payload) private void OnMemPoolMessageReceived() { - foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, Blockchain.Singleton.GetMemoryPool().Select(p => p.Hash).ToArray())) + foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => p.Hash).ToArray())) Context.Parent.Tell(Message.Create("inv", payload)); } diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index 0c090c770f..0f22ad4851 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -298,7 +298,20 @@ private JObject Process(string method, JArray _params) return json; } case "getrawmempool": - return new JArray(Blockchain.Singleton.GetMemoryPool().Select(p => (JObject)p.Hash.ToString())); + { + bool shouldGetUnverified = _params.Count >= 1 && _params[0].AsBooleanOrDefault(false); + if (!shouldGetUnverified) + return new JArray(Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => (JObject)p.Hash.ToString())); + + JObject json = new JObject(); + json["height"] = Blockchain.Singleton.Height; + Blockchain.Singleton.MemPool.GetVerifiedAndUnverifiedTransactions( + out IEnumerable verifiedTransactions, + out IEnumerable unverifiedTransactions); + json["verified"] = new JArray(verifiedTransactions.Select(p => (JObject) p.Hash.ToString())); + json["unverified"] = new JArray(unverifiedTransactions.Select(p => (JObject) p.Hash.ToString())); + return json; + } case "getrawtransaction": { UInt256 hash = UInt256.Parse(_params[0].AsString()); diff --git a/neo/Plugins/IPolicyPlugin.cs b/neo/Plugins/IPolicyPlugin.cs index 812d418ee2..9952cf2532 100644 --- a/neo/Plugins/IPolicyPlugin.cs +++ b/neo/Plugins/IPolicyPlugin.cs @@ -7,5 +7,7 @@ public interface IPolicyPlugin { bool FilterForMemoryPool(Transaction tx); IEnumerable FilterForBlock(IEnumerable transactions); + int MaxTxPerBlock { get; } + int MaxLowPriorityTxPerBlock { get; } } } From ff174f79d72065c2f772d062e82f0235d4d2efaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Tue, 15 Jan 2019 13:30:10 -0200 Subject: [PATCH 11/27] Policy filter GetRelayResult message (#543) * Policy filter GetRelayResult message * adding fixed numbering for return codes * Removed enum fixed values --- neo/Network/RPC/RpcServer.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index 0f22ad4851..5a12f190ba 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; @@ -127,8 +127,10 @@ private static JObject GetRelayResult(RelayResultReason reason) throw new RpcException(-503, "The block cannot be validated."); case RelayResultReason.Invalid: throw new RpcException(-504, "Block or transaction validation failed."); + case RelayResultReason.PolicyFail: + throw new RpcException(-505, "One of the Policy filters failed."); default: - throw new RpcException(-500, "Unkown error."); + throw new RpcException(-500, "Unknown error."); } } From 50a7a84f1d14fbbd4dee3e52a8c0737b9815bd68 Mon Sep 17 00:00:00 2001 From: jsolman Date: Tue, 15 Jan 2019 19:38:45 -0800 Subject: [PATCH 12/27] Add some initial MemoryPool unit tests. Fix bug when Persisting the GenesisBlock (#549) --- neo.UnitTests/TestDataCache.cs | 16 ++- neo.UnitTests/UT_MemoryPool.cs | 214 +++++++++++++++++++++++++++++++++ neo/Ledger/MemoryPool.cs | 18 ++- 3 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 neo.UnitTests/UT_MemoryPool.cs diff --git a/neo.UnitTests/TestDataCache.cs b/neo.UnitTests/TestDataCache.cs index d2d3afdabf..68f3b07da7 100644 --- a/neo.UnitTests/TestDataCache.cs +++ b/neo.UnitTests/TestDataCache.cs @@ -10,6 +10,17 @@ public class TestDataCache : DataCache where TKey : IEquatable, ISerializable where TValue : class, ICloneable, ISerializable, new() { + private readonly TValue _defaultValue; + + public TestDataCache() + { + _defaultValue = null; + } + + public TestDataCache(TValue defaultValue) + { + this._defaultValue = defaultValue; + } public override void DeleteInternal(TKey key) { } @@ -25,12 +36,13 @@ protected override void AddInternal(TKey key, TValue value) protected override TValue GetInternal(TKey key) { - throw new NotImplementedException(); + if (_defaultValue == null) throw new NotImplementedException(); + return _defaultValue; } protected override TValue TryGetInternal(TKey key) { - return null; + return _defaultValue; } protected override void UpdateInternal(TKey key, TValue value) diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs new file mode 100644 index 0000000000..e9094ece93 --- /dev/null +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -0,0 +1,214 @@ +using System; +using System.Linq; +using System.Reflection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Ledger; +using FluentAssertions; +using Neo.Cryptography.ECC; +using Neo.IO.Wrappers; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_MemoryPool + { + private static NeoSystem TheNeoSystem; + + private readonly Random _random = new Random(); + + private MemoryPool _unit; + + [TestInitialize] + public void TestSetup() + { + if (TheNeoSystem == null) + { + var mockSnapshot = new Mock(); + mockSnapshot.SetupGet(p => p.Blocks).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Transactions).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Accounts).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.UnspentCoins).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.SpentCoins).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Validators).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Assets).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Contracts).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Storages).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.HeaderHashList) + .Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.ValidatorsCount).Returns(new TestMetaDataCache()); + mockSnapshot.SetupGet(p => p.BlockHashIndex).Returns(new TestMetaDataCache()); + mockSnapshot.SetupGet(p => p.HeaderHashIndex).Returns(new TestMetaDataCache()); + + var mockStore = new Mock(); + + var defaultTx = CreateRandomHashInvocationTransaction(); + defaultTx.Outputs = new TransactionOutput[1]; + defaultTx.Outputs[0] = new TransactionOutput + { + AssetId = Blockchain.UtilityToken.Hash, + Value = new Fixed8(1000000), // 0.001 GAS (enough to be a high priority TX + ScriptHash = UInt160.Zero // doesn't matter for our purposes. + }; + + mockStore.Setup(p => p.GetBlocks()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetTransactions()).Returns(new TestDataCache( + new TransactionState + { + BlockIndex = 1, + Transaction = defaultTx + })); + + mockStore.Setup(p => p.GetAccounts()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetUnspentCoins()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetSpentCoins()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetValidators()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetAssets()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetContracts()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetStorages()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetHeaderHashList()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetValidatorsCount()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetBlockHashIndex()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetHeaderHashIndex()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetSnapshot()).Returns(mockSnapshot.Object); + + Console.WriteLine("initialize NeoSystem"); + TheNeoSystem = new NeoSystem(mockStore.Object); // new Mock(mockStore.Object); + } + + // Create a MemoryPool with capacity of 100 + _unit = new MemoryPool(TheNeoSystem, 100); + + // Verify capacity equals the amount specified + _unit.Capacity.ShouldBeEquivalentTo(100); + + _unit.VerifiedCount.ShouldBeEquivalentTo(0); + _unit.UnVerifiedCount.ShouldBeEquivalentTo(0); + _unit.Count.ShouldBeEquivalentTo(0); + } + + private Transaction CreateRandomHashInvocationTransaction() + { + var tx = new InvocationTransaction(); + var randomBytes = new byte[16]; + _random.NextBytes(randomBytes); + tx.Script = randomBytes; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + // Force getting the references + // Console.WriteLine($"Reference Count: {tx.References.Count}"); + return tx; + } + + private Transaction CreateMockHighPriorityTransaction() + { + var tx = CreateRandomHashInvocationTransaction(); + tx.Inputs = new CoinReference[1]; + // Any input will trigger reading the transaction output and get our mocked transaction output. + tx.Inputs[0] = new CoinReference + { + PrevHash = UInt256.Zero, + PrevIndex = 0 + }; + return tx; + } + + + private Transaction CreateMockLowPriorityTransaction() + { + return CreateRandomHashInvocationTransaction(); + } + + private void AddTransactions(int count, bool isHighPriority=false) + { + for (int i = 0; i < count; i++) + { + var lowPrioTx = isHighPriority ? CreateMockHighPriorityTransaction(): CreateMockLowPriorityTransaction(); + Console.WriteLine($"created tx: {lowPrioTx.Hash}"); + _unit.TryAdd(lowPrioTx.Hash, lowPrioTx); + } + } + + private void AddLowPriorityTransactions(int count) => AddTransactions(count); + public void AddHighPriorityTransactions(int count) => AddTransactions(count, true); + + + [TestMethod] + public void LowPriorityCapacityTest() + { + // Add over the capacity items, verify that the verified count increases each time + AddLowPriorityTransactions(50); + _unit.VerifiedCount.ShouldBeEquivalentTo(50); + AddLowPriorityTransactions(51); + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}"); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(100); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + + _unit.VerifiedCount.ShouldBeEquivalentTo(100); + _unit.UnVerifiedCount.ShouldBeEquivalentTo(0); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void HighPriorityCapacityTest() + { + // Add over the capacity items, verify that the verified count increases each time + AddHighPriorityTransactions(101); + + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}"); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(100); + + _unit.VerifiedCount.ShouldBeEquivalentTo(100); + _unit.UnVerifiedCount.ShouldBeEquivalentTo(0); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void HighPriorityPushesOutLowPriority() + { + // Add over the capacity items, verify that the verified count increases each time + AddLowPriorityTransactions(70); + AddHighPriorityTransactions(40); + + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}"); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(40); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void LowPriorityDoesNotPushOutHighPrority() + { + AddHighPriorityTransactions(70); + AddLowPriorityTransactions(40); + + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(30); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(70); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void BlockPersistMovesTxToUnverified() + { + AddLowPriorityTransactions(30); + AddHighPriorityTransactions(70); + + + var block = new Block + { + Transactions = _unit.GetSortedVerifiedTransactions().Take(10) + .Concat(_unit.GetSortedVerifiedTransactions().Where(x => x.IsLowPriority).Take(5)).ToArray() + }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(25); + } + } +} \ No newline at end of file diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 1d05679934..67de6f05a1 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -73,13 +73,14 @@ public int CompareTo(PoolItem otherItem) /// private readonly Dictionary _unsortedTransactions = new Dictionary(); /// - /// Stores the verified low priority sorted transactions currently in the pool. - /// - private readonly SortedSet _sortedLowPrioTransactions = new SortedSet(); - /// /// Stores the verified high priority sorted transactins currently in the pool. /// private readonly SortedSet _sortedHighPrioTransactions = new SortedSet(); + /// + /// Stores the verified low priority sorted transactions currently in the pool. + /// + private readonly SortedSet _sortedLowPrioTransactions = new SortedSet(); + /// /// Store the unverified transactions currently in the pool. @@ -92,6 +93,13 @@ public int CompareTo(PoolItem otherItem) private readonly SortedSet _unverifiedSortedHighPriorityTransactions = new SortedSet(); private readonly SortedSet _unverifiedSortedLowPriorityTransactions = new SortedSet(); + // internal methods to aid in unit testing + internal int SortedHighPrioTxCount => _sortedHighPrioTransactions.Count; + internal int SortedLowPrioTxCount => _sortedLowPrioTransactions.Count; + internal int UnverifiedSortedHighPrioTxCount => _unverifiedSortedHighPriorityTransactions.Count; + internal int UnverifiedSortedLowPrioTxCount => _unverifiedSortedLowPriorityTransactions.Count; + + private int _maxTxPerBlock; private int _maxLowPriorityTxPerBlock; @@ -401,7 +409,7 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) // If we know about headers of future blocks, no point in verifying transactions from the unverified tx pool // until we get caught up. - if (block.Index < Blockchain.Singleton.HeaderHeight) + if (block.Index > 0 && block.Index < Blockchain.Singleton.HeaderHeight) return; if (Plugin.Policies.Count == 0) From ab8bb4d5277b0d86d49c88dfca11368272b74d18 Mon Sep 17 00:00:00 2001 From: jsolman Date: Thu, 17 Jan 2019 07:54:24 -0800 Subject: [PATCH 13/27] More MemoryPool Unit Tests. Improve Re-broadcast back-off to an increasing linear formula. (#554) --- neo.UnitTests/UT_MemoryPool.cs | 106 ++++++++++++++++++++++++++++----- neo/Ledger/MemoryPool.cs | 4 ++ 2 files changed, 96 insertions(+), 14 deletions(-) diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs index e9094ece93..d98efc43aa 100644 --- a/neo.UnitTests/UT_MemoryPool.cs +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -1,6 +1,6 @@ using System; +using System.Collections.Generic; using System.Linq; -using System.Reflection; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Neo.Ledger; @@ -17,7 +17,7 @@ public class UT_MemoryPool { private static NeoSystem TheNeoSystem; - private readonly Random _random = new Random(); + private readonly Random _random = new Random(1337); // use fixed seed for guaranteed determinism private MemoryPool _unit; @@ -44,7 +44,7 @@ public void TestSetup() var mockStore = new Mock(); - var defaultTx = CreateRandomHashInvocationTransaction(); + var defaultTx = CreateRandomHashInvocationMockTransaction().Object; defaultTx.Outputs = new TransactionOutput[1]; defaultTx.Outputs[0] = new TransactionOutput { @@ -89,9 +89,12 @@ public void TestSetup() _unit.Count.ShouldBeEquivalentTo(0); } - private Transaction CreateRandomHashInvocationTransaction() + private Mock CreateRandomHashInvocationMockTransaction() { - var tx = new InvocationTransaction(); + var mockTx = new Mock(); + mockTx.CallBase = true; + mockTx.Setup(p => p.Verify(It.IsAny(), It.IsAny>())).Returns(true); + var tx = mockTx.Object; var randomBytes = new byte[16]; _random.NextBytes(randomBytes); tx.Script = randomBytes; @@ -99,14 +102,15 @@ private Transaction CreateRandomHashInvocationTransaction() tx.Inputs = new CoinReference[0]; tx.Outputs = new TransactionOutput[0]; tx.Witnesses = new Witness[0]; - // Force getting the references - // Console.WriteLine($"Reference Count: {tx.References.Count}"); - return tx; + + return mockTx; } private Transaction CreateMockHighPriorityTransaction() { - var tx = CreateRandomHashInvocationTransaction(); + var mockTx = CreateRandomHashInvocationMockTransaction(); + mockTx.SetupGet(p => p.NetworkFee).Returns(Fixed8.FromDecimal(0.001m)); + var tx = mockTx.Object; tx.Inputs = new CoinReference[1]; // Any input will trigger reading the transaction output and get our mocked transaction output. tx.Inputs[0] = new CoinReference @@ -120,7 +124,8 @@ private Transaction CreateMockHighPriorityTransaction() private Transaction CreateMockLowPriorityTransaction() { - return CreateRandomHashInvocationTransaction(); + var mockTx = CreateRandomHashInvocationMockTransaction(); + return mockTx.Object; } private void AddTransactions(int count, bool isHighPriority=false) @@ -136,7 +141,6 @@ private void AddTransactions(int count, bool isHighPriority=false) private void AddLowPriorityTransactions(int count) => AddTransactions(count); public void AddHighPriorityTransactions(int count) => AddTransactions(count, true); - [TestMethod] public void LowPriorityCapacityTest() { @@ -193,11 +197,13 @@ public void LowPriorityDoesNotPushOutHighPrority() } [TestMethod] - public void BlockPersistMovesTxToUnverified() + public void BlockPersistMovesTxToUnverifiedAndReverification() { - AddLowPriorityTransactions(30); AddHighPriorityTransactions(70); + AddLowPriorityTransactions(30); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(70); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(30); var block = new Block { @@ -205,10 +211,82 @@ public void BlockPersistMovesTxToUnverified() .Concat(_unit.GetSortedVerifiedTransactions().Where(x => x.IsLowPriority).Take(5)).ToArray() }; _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); - _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(60); _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(25); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(9); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(1); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(51); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(24); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(18); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(2); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(42); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(23); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(27); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(3); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(33); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(22); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(36); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(4); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(24); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(21); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(45); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(5); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(15); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(20); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(54); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(6); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(6); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(19); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(10); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(15); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(20); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(5); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(25); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(0); + } + + [TestMethod] + public void CapacityTestWithUnverifiedHighProirtyTransactions() + { + // Verify that unverified high priority transactions will not be pushed out of the queue by incoming + // low priority transactions + + // Fill pool with high priority transactions + AddHighPriorityTransactions(99); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + + _unit.CanTransactionFitInPool(CreateMockLowPriorityTransaction()).ShouldBeEquivalentTo(true); + AddHighPriorityTransactions(1); + _unit.CanTransactionFitInPool(CreateMockLowPriorityTransaction()).ShouldBeEquivalentTo(false); } } } \ No newline at end of file diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 67de6f05a1..4b5b1910cf 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -48,6 +48,7 @@ public int CompareTo(PoolItem otherItem) // Allow reverified transactions to be rebroadcast if it has been this many block times since last broadcast. private const int BlocksTillRebroadcastLowPriorityPoolTx = 30; private const int BlocksTillRebroadcastHighPriorityPoolTx = 10; + private int RebroadcastMultiplierThreshold => Capacity / 10; private static readonly double MaxSecondsToReverifyHighPrioTx = (double) Blockchain.SecondsPerBlock / 3; private static readonly double MaxSecondsToReverifyLowPrioTx = (double) Blockchain.SecondsPerBlock / 5; @@ -448,6 +449,9 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) int blocksTillRebroadcast = unverifiedSortedTxPool == _sortedHighPrioTransactions ? BlocksTillRebroadcastHighPriorityPoolTx : BlocksTillRebroadcastLowPriorityPoolTx; + if (Count > RebroadcastMultiplierThreshold) + blocksTillRebroadcast = blocksTillRebroadcast * Count / RebroadcastMultiplierThreshold; + var rebroadcastCutOffTime = DateTime.UtcNow.AddSeconds( -Blockchain.SecondsPerBlock * blocksTillRebroadcast); foreach (PoolItem item in reverifiedItems) From 0e5434c6296f52dd02f1b13326a34468364c1b3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Thu, 17 Jan 2019 17:12:23 -0200 Subject: [PATCH 14/27] Ensuring Object Reference check of SortedSets for speed-up (#557) --- neo/Ledger/MemoryPool.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 4b5b1910cf..7fdbafe187 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -279,7 +279,7 @@ private PoolItem GetLowestFeeTransaction(out Dictionary unsor if (minItem != null) { - unsortedTxPool = sortedPool == _unverifiedSortedLowPriorityTransactions + unsortedTxPool = Object.ReferenceEquals(sortedPool, _unverifiedSortedLowPriorityTransactions) ? _unverifiedTransactions : _unsortedTransactions; return minItem; } @@ -291,7 +291,7 @@ private PoolItem GetLowestFeeTransaction(out Dictionary unsor } finally { - unsortedTxPool = sortedPool == _unverifiedSortedHighPriorityTransactions + unsortedTxPool = Object.ReferenceEquals(sortedPool, _unverifiedSortedHighPriorityTransactions) ? _unverifiedTransactions : _unsortedTransactions; } } @@ -446,7 +446,7 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) _txRwLock.EnterWriteLock(); try { - int blocksTillRebroadcast = unverifiedSortedTxPool == _sortedHighPrioTransactions + int blocksTillRebroadcast = Object.ReferenceEquals(unverifiedSortedTxPool, _sortedHighPrioTransactions) ? BlocksTillRebroadcastHighPriorityPoolTx : BlocksTillRebroadcastLowPriorityPoolTx; if (Count > RebroadcastMultiplierThreshold) From 1aeaf86b2578702ba1c437783bb91c20981c6819 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Thu, 17 Jan 2019 18:04:12 -0200 Subject: [PATCH 15/27] Minor comments update on Mempool class (#556) --- neo/Ledger/MemoryPool.cs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 7fdbafe187..521701d0de 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -45,7 +45,7 @@ public int CompareTo(PoolItem otherItem) } } - // Allow reverified transactions to be rebroadcast if it has been this many block times since last broadcast. + // Allow a reverified transaction to be rebroadcasted if it has been this many block times since last broadcast. private const int BlocksTillRebroadcastLowPriorityPoolTx = 30; private const int BlocksTillRebroadcastHighPriorityPoolTx = 10; private int RebroadcastMultiplierThreshold => Capacity / 10; @@ -82,7 +82,6 @@ public int CompareTo(PoolItem otherItem) /// private readonly SortedSet _sortedLowPrioTransactions = new SortedSet(); - /// /// Store the unverified transactions currently in the pool. /// @@ -94,13 +93,12 @@ public int CompareTo(PoolItem otherItem) private readonly SortedSet _unverifiedSortedHighPriorityTransactions = new SortedSet(); private readonly SortedSet _unverifiedSortedLowPriorityTransactions = new SortedSet(); - // internal methods to aid in unit testing + // Internal methods to aid in unit testing internal int SortedHighPrioTxCount => _sortedHighPrioTransactions.Count; internal int SortedLowPrioTxCount => _sortedLowPrioTransactions.Count; internal int UnverifiedSortedHighPrioTxCount => _unverifiedSortedHighPriorityTransactions.Count; internal int UnverifiedSortedLowPrioTxCount => _unverifiedSortedLowPriorityTransactions.Count; - private int _maxTxPerBlock; private int _maxLowPriorityTxPerBlock; @@ -429,15 +427,15 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) SortedSet unverifiedSortedTxPool, int count, double secondsTimeout, Snapshot snapshot) { DateTime reverifyCutOffTimeStamp = DateTime.UtcNow.AddSeconds(secondsTimeout); - List reverifiedItems = new List(count); List invalidItems = new List(); + + // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end. foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) { - // Re-verify the top fee max high priority transactions that can be verified in a block if (item.Transaction.Verify(snapshot, _unsortedTransactions.Select(p => p.Value.Transaction))) reverifiedItems.Add(item); - else // Transaction no longer valid -- will be removed from unverifiedTxPool. + else // Transaction no longer valid -- it will be removed from unverifiedTxPool. invalidItems.Add(item); if (DateTime.UtcNow > reverifyCutOffTimeStamp) break; From b222995493e20c202eaaec70fac51b48079f9bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Fri, 18 Jan 2019 00:37:55 -0200 Subject: [PATCH 16/27] Update MemoryPool Unit Test to add random fees to Mock Transactions (#558) --- neo.UnitTests/UT_MemoryPool.cs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs index d98efc43aa..ad60229c4e 100644 --- a/neo.UnitTests/UT_MemoryPool.cs +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -49,7 +49,7 @@ public void TestSetup() defaultTx.Outputs[0] = new TransactionOutput { AssetId = Blockchain.UtilityToken.Hash, - Value = new Fixed8(1000000), // 0.001 GAS (enough to be a high priority TX + Value = new Fixed8(1000000), ScriptHash = UInt160.Zero // doesn't matter for our purposes. }; @@ -106,10 +106,18 @@ private Mock CreateRandomHashInvocationMockTransaction() return mockTx; } + long LongRandom(long min, long max, Random rand) + { + // Only returns positive random long values. + long longRand = (long) rand.NextBigInteger(63); + return longRand % (max - min) + min; + } + private Transaction CreateMockHighPriorityTransaction() { var mockTx = CreateRandomHashInvocationMockTransaction(); - mockTx.SetupGet(p => p.NetworkFee).Returns(Fixed8.FromDecimal(0.001m)); + long rNetFee = LongRandom(100000, 100000000, _random); // [0.001,1]) GAS (enough to be a high priority TX) + mockTx.SetupGet(p => p.NetworkFee).Returns(new Fixed8(rNetFee)); var tx = mockTx.Object; tx.Inputs = new CoinReference[1]; // Any input will trigger reading the transaction output and get our mocked transaction output. @@ -121,10 +129,11 @@ private Transaction CreateMockHighPriorityTransaction() return tx; } - private Transaction CreateMockLowPriorityTransaction() { var mockTx = CreateRandomHashInvocationMockTransaction(); + long rNetFee = LongRandom(0, 100000, _random); // [0,0.001] GAS a fee lower than the threshold of 0.001 GAS (not enough to be a high priority TX) + mockTx.SetupGet(p => p.NetworkFee).Returns(new Fixed8(rNetFee)); return mockTx.Object; } @@ -132,9 +141,9 @@ private void AddTransactions(int count, bool isHighPriority=false) { for (int i = 0; i < count; i++) { - var lowPrioTx = isHighPriority ? CreateMockHighPriorityTransaction(): CreateMockLowPriorityTransaction(); - Console.WriteLine($"created tx: {lowPrioTx.Hash}"); - _unit.TryAdd(lowPrioTx.Hash, lowPrioTx); + var txToAdd = isHighPriority ? CreateMockHighPriorityTransaction(): CreateMockLowPriorityTransaction(); + Console.WriteLine($"created tx: {txToAdd.Hash}"); + _unit.TryAdd(txToAdd.Hash, txToAdd); } } @@ -289,4 +298,4 @@ public void CapacityTestWithUnverifiedHighProirtyTransactions() _unit.CanTransactionFitInPool(CreateMockLowPriorityTransaction()).ShouldBeEquivalentTo(false); } } -} \ No newline at end of file +} From 54ff67d62578b4d6536a837ae7e19c1382675f8b Mon Sep 17 00:00:00 2001 From: jsolman Date: Fri, 18 Jan 2019 05:23:06 -0800 Subject: [PATCH 17/27] Add Unit Test for MemoryPool sort order. Fixed sort order to return descending. (#559) * Add unit test to verify memory pool sort order and reverification order. Fixed sort order bug. * VerifyCanTransactionFitInPool works as intended. Also inadvertantly verified GetLowestFeeTransaction() works. --- neo.UnitTests/UT_MemoryPool.cs | 123 +++++++++++++++++++++++++++++---- neo/Ledger/MemoryPool.cs | 13 ++-- 2 files changed, 117 insertions(+), 19 deletions(-) diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs index ad60229c4e..b5167e9b16 100644 --- a/neo.UnitTests/UT_MemoryPool.cs +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -113,28 +113,34 @@ long LongRandom(long min, long max, Random rand) return longRand % (max - min) + min; } - private Transaction CreateMockHighPriorityTransaction() + private Transaction CreateMockTransactionWithFee(long fee) { var mockTx = CreateRandomHashInvocationMockTransaction(); - long rNetFee = LongRandom(100000, 100000000, _random); // [0.001,1]) GAS (enough to be a high priority TX) - mockTx.SetupGet(p => p.NetworkFee).Returns(new Fixed8(rNetFee)); + mockTx.SetupGet(p => p.NetworkFee).Returns(new Fixed8(fee)); var tx = mockTx.Object; - tx.Inputs = new CoinReference[1]; - // Any input will trigger reading the transaction output and get our mocked transaction output. - tx.Inputs[0] = new CoinReference + if (fee > 0) { - PrevHash = UInt256.Zero, - PrevIndex = 0 - }; + tx.Inputs = new CoinReference[1]; + // Any input will trigger reading the transaction output and get our mocked transaction output. + tx.Inputs[0] = new CoinReference + { + PrevHash = UInt256.Zero, + PrevIndex = 0 + }; + } return tx; } + private Transaction CreateMockHighPriorityTransaction() + { + return CreateMockTransactionWithFee(LongRandom(100000, 100000000, _random)); + } + private Transaction CreateMockLowPriorityTransaction() { - var mockTx = CreateRandomHashInvocationMockTransaction(); - long rNetFee = LongRandom(0, 100000, _random); // [0,0.001] GAS a fee lower than the threshold of 0.001 GAS (not enough to be a high priority TX) - mockTx.SetupGet(p => p.NetworkFee).Returns(new Fixed8(rNetFee)); - return mockTx.Object; + long rNetFee = LongRandom(0, 100000, _random); + // [0,0.001] GAS a fee lower than the threshold of 0.001 GAS (not enough to be a high priority TX) + return CreateMockTransactionWithFee(rNetFee); } private void AddTransactions(int count, bool isHighPriority=false) @@ -280,6 +286,97 @@ public void BlockPersistMovesTxToUnverifiedAndReverification() _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(0); } + private void verifyTransactionsSortedDescending(IEnumerable transactions) + { + Transaction lastTransaction = null; + foreach (var tx in transactions) + { + if (lastTransaction != null) + { + if (lastTransaction.FeePerByte == tx.FeePerByte) + { + if (lastTransaction.NetworkFee == tx.NetworkFee) + lastTransaction.Hash.Should().BeGreaterThan(tx.Hash); + else + lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); + } + else + { + lastTransaction.FeePerByte.Should().BeGreaterThan(tx.FeePerByte); + } + } + lastTransaction = tx; + } + } + + [TestMethod] + public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() + { + AddLowPriorityTransactions(50); + AddHighPriorityTransactions(50); + + var sortedVerifiedTxs = _unit.GetSortedVerifiedTransactions().ToList(); + // verify all 100 transactions are returned in sorted order + sortedVerifiedTxs.Count.ShouldBeEquivalentTo(100); + verifyTransactionsSortedDescending(sortedVerifiedTxs); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(50); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(50); + + // We can verify the order they are re-verified by reverifying 2 at a time + while (_unit.UnVerifiedCount > 0) + { + _unit.GetVerifiedAndUnverifiedTransactions(out IEnumerable sortedVerifiedTransactions, + out IEnumerable sortedUnverifiedTransactions); + sortedVerifiedTransactions.Count().ShouldBeEquivalentTo(0); + var sortedUnverifiedArray = sortedUnverifiedTransactions.ToArray(); + verifyTransactionsSortedDescending(sortedUnverifiedArray); + var maxHighPriorityTransaction = sortedUnverifiedArray.First(); + var maxLowPriorityTransaction = sortedUnverifiedArray.First(tx => tx.IsLowPriority); + + // reverify 1 high priority and 1 low priority transaction + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(2, Blockchain.Singleton.GetSnapshot()); + var verifiedTxs = _unit.GetSortedVerifiedTransactions().ToArray(); + verifiedTxs.Length.ShouldBeEquivalentTo(2); + verifiedTxs[0].ShouldBeEquivalentTo(maxHighPriorityTransaction); + verifiedTxs[1].ShouldBeEquivalentTo(maxLowPriorityTransaction); + var blockWith2Tx = new Block { Transactions = new Transaction[2] { maxHighPriorityTransaction, maxLowPriorityTransaction }}; + // verify and remove the 2 transactions from the verified pool + _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + } + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(0); + } + + void VerifyCapacityThresholdForAttemptingToAddATransaction() + { + var sortedVerified = _unit.GetSortedVerifiedTransactions().ToArray(); + + var txBarelyWontFit = CreateMockTransactionWithFee(sortedVerified.Last().NetworkFee.GetData() - 1); + _unit.CanTransactionFitInPool(txBarelyWontFit).ShouldBeEquivalentTo(false); + var txBarelyFits = CreateMockTransactionWithFee(sortedVerified.Last().NetworkFee.GetData() + 1); + _unit.CanTransactionFitInPool(txBarelyFits).ShouldBeEquivalentTo(true); + } + + [TestMethod] + public void VerifyCanTransactionFitInPoolWorksAsIntended() + { + AddLowPriorityTransactions(100); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddHighPriorityTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddHighPriorityTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + } + [TestMethod] public void CapacityTestWithUnverifiedHighProirtyTransactions() { diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 521701d0de..1b89e5d1b7 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -226,9 +226,10 @@ public IEnumerable GetVerifiedTransactions() _txRwLock.EnterReadLock(); try { - verifiedTransactions = _sortedHighPrioTransactions.Select(p => p.Transaction) - .Concat(_sortedLowPrioTransactions.Select(p => p.Transaction)).ToArray(); - unverifiedTransactions = _unverifiedTransactions.Select(p => p.Value.Transaction).ToArray(); + verifiedTransactions = _sortedHighPrioTransactions.Reverse().Select(p => p.Transaction) + .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Transaction)).ToArray(); + unverifiedTransactions = _unverifiedSortedHighPriorityTransactions.Reverse().Select(p => p.Transaction) + .Concat(_unverifiedSortedLowPriorityTransactions.Reverse().Select(p => p.Transaction)).ToArray(); } finally { @@ -241,8 +242,8 @@ public IEnumerable GetSortedVerifiedTransactions() _txRwLock.EnterReadLock(); try { - return _sortedHighPrioTransactions.Select(p => p.Transaction) - .Concat(_sortedLowPrioTransactions.Select(p => p.Transaction)) + return _sortedHighPrioTransactions.Reverse().Select(p => p.Transaction) + .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Transaction)) .ToArray(); } finally @@ -429,7 +430,7 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) DateTime reverifyCutOffTimeStamp = DateTime.UtcNow.AddSeconds(secondsTimeout); List reverifiedItems = new List(count); List invalidItems = new List(); - + // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end. foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) { From 6849ac2bbd8e683baa6ccb3ad5dbb90fc294b778 Mon Sep 17 00:00:00 2001 From: Igor Machado Coelho Date: Fri, 18 Jan 2019 12:11:16 -0200 Subject: [PATCH 18/27] Benchmark structure for UInt classes (#553) * basic benchmark structure for UInt classes * commented code2 from lights for now * updated tests. all seem correct now * Switch to using a benchmark method taking a method delegate to benchmark. * Make pass. * 1 million iterations. * Switch to ulong for the 4th option, and it is still the same speed. * fix test data for 50% equal data * make test pass * neo.UnitTests/UT_UIntBenchmarks.cs * neo.UnitTests/UT_UIntBenchmarks.cs * Base 20 - UInt160 tests * neo.UnitTests/UT_UIntBenchmarks.cs * inlined 160 * complete tests with UInt256 and UInt160 * neo.UnitTests/UT_UIntBenchmarks.cs * Lights division calculation --- neo.UnitTests/UT_UIntBenchmarks.cs | 333 +++++++++++++++++++++++++++++ neo.UnitTests/neo.UnitTests.csproj | 1 + 2 files changed, 334 insertions(+) create mode 100644 neo.UnitTests/UT_UIntBenchmarks.cs diff --git a/neo.UnitTests/UT_UIntBenchmarks.cs b/neo.UnitTests/UT_UIntBenchmarks.cs new file mode 100644 index 0000000000..1c388354a5 --- /dev/null +++ b/neo.UnitTests/UT_UIntBenchmarks.cs @@ -0,0 +1,333 @@ +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.Cryptography.ECC; +using Neo.IO; +using Neo.Ledger; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Diagnostics; +using System; +//using System.Runtime.CompilerServices.Unsafe; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_UIntBenchmarks + { + int MAX_TESTS = 1000000; // 1 million + + byte[][] base_32_1; + byte[][] base_32_2; + byte[][] base_20_1; + byte[][] base_20_2; + + private Random random; + + [TestInitialize] + public void TestSetup() + { + int SEED = 123456789; + random = new Random(SEED); + + base_32_1 = new byte[MAX_TESTS][]; + base_32_2 = new byte[MAX_TESTS][]; + base_20_1 = new byte[MAX_TESTS][]; + base_20_2 = new byte[MAX_TESTS][]; + + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i= 0; i--) + { + if (x[i] > y[i]) + return 1; + if (x[i] < y[i]) + return -1; + } + return 0; + } + + private unsafe int code2_UInt256CompareTo(byte[] b1, byte[] b2) + { + fixed (byte* px = b1, py = b2) + { + uint* lpx = (uint*)px; + uint* lpy = (uint*)py; + for (int i = 256/32-1; i >= 0; i--) + { + if (lpx[i] > lpy[i]) + return 1; + if (lpx[i] < lpy[i]) + return -1; + } + } + return 0; + } + + private unsafe int code3_UInt256CompareTo(byte[] b1, byte[] b2) + { + fixed (byte* px = b1, py = b2) + { + ulong* lpx = (ulong*)px; + ulong* lpy = (ulong*)py; + for (int i = 256/64-1; i >= 0; i--) + { + if (lpx[i] > lpy[i]) + return 1; + if (lpx[i] < lpy[i]) + return -1; + } + } + return 0; + } + private int code1_UInt160CompareTo(byte[] b1, byte[] b2) + { + byte[] x = b1; + byte[] y = b2; + for (int i = x.Length - 1; i >= 0; i--) + { + if (x[i] > y[i]) + return 1; + if (x[i] < y[i]) + return -1; + } + return 0; + } + + private unsafe int code2_UInt160CompareTo(byte[] b1, byte[] b2) + { + fixed (byte* px = b1, py = b2) + { + uint* lpx = (uint*)px; + uint* lpy = (uint*)py; + for (int i = 160/32-1; i >= 0; i--) + { + if (lpx[i] > lpy[i]) + return 1; + if (lpx[i] < lpy[i]) + return -1; + } + } + return 0; + } + + private unsafe int code3_UInt160CompareTo(byte[] b1, byte[] b2) + { + // LSB -----------------> MSB + // -------------------------- + // | 8B | 8B | 4B | + // -------------------------- + // 0l 1l 4i + // -------------------------- + fixed (byte* px = b1, py = b2) + { + uint* ipx = (uint*)px; + uint* ipy = (uint*)py; + if (ipx[4] > ipy[4]) + return 1; + if (ipx[4] < ipy[4]) + return -1; + + ulong* lpx = (ulong*)px; + ulong* lpy = (ulong*)py; + if (lpx[1] > lpy[1]) + return 1; + if (lpx[1] < lpy[1]) + return -1; + if (lpx[0] > lpy[0]) + return 1; + if (lpx[0] < lpy[0]) + return -1; + } + return 0; + } + + } +} diff --git a/neo.UnitTests/neo.UnitTests.csproj b/neo.UnitTests/neo.UnitTests.csproj index 3d75fd8e28..158bfba690 100644 --- a/neo.UnitTests/neo.UnitTests.csproj +++ b/neo.UnitTests/neo.UnitTests.csproj @@ -5,6 +5,7 @@ netcoreapp2.0 Neo.UnitTests Neo.UnitTests + true From e5d012e87e605835df156575043682a75b4e66e9 Mon Sep 17 00:00:00 2001 From: jsolman Date: Fri, 18 Jan 2019 08:32:28 -0800 Subject: [PATCH 19/27] Treat lower hashes as higher priority. Fix MemoryPool UT for Hash order. (#563) * Treat lower hashes as higher priority. * Fix MemoryPool UT for Hash order. * Renaming Trasanction in PoolItem for clarity. --- neo.UnitTests/UT_MemoryPool.cs | 2 +- neo/Ledger/MemoryPool.cs | 59 +++++++++++++++++----------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs index b5167e9b16..8393528bd6 100644 --- a/neo.UnitTests/UT_MemoryPool.cs +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -296,7 +296,7 @@ private void verifyTransactionsSortedDescending(IEnumerable transac if (lastTransaction.FeePerByte == tx.FeePerByte) { if (lastTransaction.NetworkFee == tx.NetworkFee) - lastTransaction.Hash.Should().BeGreaterThan(tx.Hash); + lastTransaction.Hash.Should().BeLessThan(tx.Hash); else lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); } diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 1b89e5d1b7..c6df72891a 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -16,32 +16,33 @@ public class MemoryPool : IReadOnlyCollection { private class PoolItem : IComparable { - public readonly Transaction Transaction; + public readonly Transaction Tx; public readonly DateTime Timestamp; public DateTime LastBroadcastTimestamp; public PoolItem(Transaction tx) { - Transaction = tx; + Tx = tx; Timestamp = DateTime.UtcNow; LastBroadcastTimestamp = Timestamp; } - public int CompareTo(Transaction tx) + public int CompareTo(Transaction otherTx) { - if (tx == null) return 1; - int ret = Transaction.FeePerByte.CompareTo(tx.FeePerByte); + if (otherTx == null) return 1; + // Fees sorted ascending + int ret = Tx.FeePerByte.CompareTo(otherTx.FeePerByte); if (ret != 0) return ret; - ret = Transaction.NetworkFee.CompareTo(tx.NetworkFee); + ret = Tx.NetworkFee.CompareTo(otherTx.NetworkFee); if (ret != 0) return ret; - - return Transaction.Hash.CompareTo(tx.Hash); + // Transaction hash sorted descending + return otherTx.Hash.CompareTo(Tx.Hash); } public int CompareTo(PoolItem otherItem) { if (otherItem == null) return 1; - return CompareTo(otherItem.Transaction); + return CompareTo(otherItem.Tx); } } @@ -179,7 +180,7 @@ public bool TryGetValue(UInt256 hash, out Transaction tx) { bool ret = _unsortedTransactions.TryGetValue(hash, out PoolItem item) || _unverifiedTransactions.TryGetValue(hash, out item); - tx = ret ? item.Transaction : null; + tx = ret ? item.Tx : null; return ret; } finally @@ -194,8 +195,8 @@ public IEnumerator GetEnumerator() _txRwLock.EnterReadLock(); try { - return _unsortedTransactions.Select(p => p.Value.Transaction) - .Concat(_unverifiedTransactions.Select(p => p.Value.Transaction)) + return _unsortedTransactions.Select(p => p.Value.Tx) + .Concat(_unverifiedTransactions.Select(p => p.Value.Tx)) .ToList() .GetEnumerator(); } @@ -212,7 +213,7 @@ public IEnumerable GetVerifiedTransactions() _txRwLock.EnterReadLock(); try { - return _unsortedTransactions.Select(p => p.Value.Transaction).ToArray(); + return _unsortedTransactions.Select(p => p.Value.Tx).ToArray(); } finally { @@ -226,10 +227,10 @@ public IEnumerable GetVerifiedTransactions() _txRwLock.EnterReadLock(); try { - verifiedTransactions = _sortedHighPrioTransactions.Reverse().Select(p => p.Transaction) - .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Transaction)).ToArray(); - unverifiedTransactions = _unverifiedSortedHighPriorityTransactions.Reverse().Select(p => p.Transaction) - .Concat(_unverifiedSortedLowPriorityTransactions.Reverse().Select(p => p.Transaction)).ToArray(); + verifiedTransactions = _sortedHighPrioTransactions.Reverse().Select(p => p.Tx) + .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Tx)).ToArray(); + unverifiedTransactions = _unverifiedSortedHighPriorityTransactions.Reverse().Select(p => p.Tx) + .Concat(_unverifiedSortedLowPriorityTransactions.Reverse().Select(p => p.Tx)).ToArray(); } finally { @@ -242,8 +243,8 @@ public IEnumerable GetSortedVerifiedTransactions() _txRwLock.EnterReadLock(); try { - return _sortedHighPrioTransactions.Reverse().Select(p => p.Transaction) - .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Transaction)) + return _sortedHighPrioTransactions.Reverse().Select(p => p.Tx) + .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Tx)) .ToArray(); } finally @@ -340,7 +341,7 @@ private void RemoveOverCapacity() { PoolItem minItem = GetLowestFeeTransaction(out var unsortedPool, out var sortedPool); - unsortedPool.Remove(minItem.Transaction.Hash); + unsortedPool.Remove(minItem.Tx.Hash); sortedPool.Remove(minItem); } } @@ -352,7 +353,7 @@ private bool TryRemoveVerified(UInt256 hash, out PoolItem item) return false; _unsortedTransactions.Remove(hash); - SortedSet pool = item.Transaction.IsLowPriority + SortedSet pool = item.Tx.IsLowPriority ? _sortedLowPrioTransactions : _sortedHighPrioTransactions; pool.Remove(item); return true; @@ -365,7 +366,7 @@ private bool TryRemoveUnVerified(UInt256 hash, out PoolItem item) return false; _unverifiedTransactions.Remove(hash); - SortedSet pool = item.Transaction.IsLowPriority + SortedSet pool = item.Tx.IsLowPriority ? _unverifiedSortedLowPriorityTransactions : _unverifiedSortedHighPriorityTransactions; pool.Remove(item); return true; @@ -387,13 +388,13 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) // Add all the previously verified transactions back to the unverified transactions foreach (PoolItem item in _sortedHighPrioTransactions) { - if (_unverifiedTransactions.TryAdd(item.Transaction.Hash, item)) + if (_unverifiedTransactions.TryAdd(item.Tx.Hash, item)) _unverifiedSortedHighPriorityTransactions.Add(item); } foreach (PoolItem item in _sortedLowPrioTransactions) { - if (_unverifiedTransactions.TryAdd(item.Transaction.Hash, item)) + if (_unverifiedTransactions.TryAdd(item.Tx.Hash, item)) _unverifiedSortedLowPriorityTransactions.Add(item); } @@ -434,7 +435,7 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end. foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) { - if (item.Transaction.Verify(snapshot, _unsortedTransactions.Select(p => p.Value.Transaction))) + if (item.Tx.Verify(snapshot, _unsortedTransactions.Select(p => p.Value.Tx))) reverifiedItems.Add(item); else // Transaction no longer valid -- it will be removed from unverifiedTxPool. invalidItems.Add(item); @@ -455,24 +456,24 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) -Blockchain.SecondsPerBlock * blocksTillRebroadcast); foreach (PoolItem item in reverifiedItems) { - if (_unsortedTransactions.TryAdd(item.Transaction.Hash, item)) + if (_unsortedTransactions.TryAdd(item.Tx.Hash, item)) { verifiedSortedTxPool.Add(item); if (item.LastBroadcastTimestamp < rebroadcastCutOffTime) { - _system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = item.Transaction }, _system.Blockchain); + _system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = item.Tx }, _system.Blockchain); item.LastBroadcastTimestamp = DateTime.UtcNow; } } - _unverifiedTransactions.Remove(item.Transaction.Hash); + _unverifiedTransactions.Remove(item.Tx.Hash); unverifiedSortedTxPool.Remove(item); } foreach (PoolItem item in invalidItems) { - _unverifiedTransactions.Remove(item.Transaction.Hash); + _unverifiedTransactions.Remove(item.Tx.Hash); unverifiedSortedTxPool.Remove(item); } } From 370d7a217e6d92407b692c054876cef253541bd1 Mon Sep 17 00:00:00 2001 From: Igor Machado Coelho Date: Fri, 18 Jan 2019 22:57:38 -0200 Subject: [PATCH 20/27] Make PoolItem independent and add PoolItem tests (#562) * make poolitem independent * Merging * Multiply by -1 * Fix other * Fix Tx * Removing -1 extra multiplication * Fix * make PoolItem internal and added test class * Update PoolItem.cs * added comments for PoolItem variables * getting time from TimeProvider to allow testing * basic test * reset time provider * Add Hash comparison * Adding time provider again and equals * Fix arithmetic * Comment on PoolItem * Update PoolItem.cs * protecting tests against TimeProvider changes on fails * reusing setup part * fixed serialization properties * Improve generation of creating mock DateTime values. Implement hash comparison tests. * Adjust comment. --- neo.UnitTests/UT_MemoryPool.cs | 3 + neo.UnitTests/UT_PoolItem.cs | 152 +++++++++++++++++++++++++++++++++ neo/Ledger/MemoryPool.cs | 32 ------- neo/Ledger/PoolItem.cs | 63 ++++++++++++++ 4 files changed, 218 insertions(+), 32 deletions(-) create mode 100644 neo.UnitTests/UT_PoolItem.cs create mode 100644 neo/Ledger/PoolItem.cs diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs index 8393528bd6..4f78b9b033 100644 --- a/neo.UnitTests/UT_MemoryPool.cs +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -24,6 +24,9 @@ public class UT_MemoryPool [TestInitialize] public void TestSetup() { + // protect against external changes on TimeProvider + TimeProvider.ResetToDefault(); + if (TheNeoSystem == null) { var mockSnapshot = new Mock(); diff --git a/neo.UnitTests/UT_PoolItem.cs b/neo.UnitTests/UT_PoolItem.cs new file mode 100644 index 0000000000..3ef28356dc --- /dev/null +++ b/neo.UnitTests/UT_PoolItem.cs @@ -0,0 +1,152 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Ledger; +using FluentAssertions; +using Neo.Network.P2P.Payloads; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_PoolItem + { + //private PoolItem uut; + private static readonly Random TestRandom = new Random(1337); // use fixed seed for guaranteed determinism + + [TestInitialize] + public void TestSetup() + { + var timeValues = new[] { + new DateTime(1968, 06, 01, 0, 0, 1, DateTimeKind.Utc), + }; + + var timeMock = new Mock(); + timeMock.SetupGet(tp => tp.UtcNow).Returns(() => timeValues[0]) + .Callback(() => timeValues[0] = timeValues[0].Add(TimeSpan.FromSeconds(1))); + TimeProvider.Current = timeMock.Object; + } + + [TestCleanup] + public void TestCleanup() + { + // important to leave TimeProvider correct + TimeProvider.ResetToDefault(); + } + + [TestMethod] + public void PoolItem_CompareTo_Fee() + { + int size1 = 50; + int netFeeSatoshi1 = 1; + var tx1 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshi1), size1); + int size2 = 50; + int netFeeSatoshi2 = 2; + var tx2 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshi2), size2); + + PoolItem pitem1 = new PoolItem(tx1.Object); + PoolItem pitem2 = new PoolItem(tx2.Object); + + Console.WriteLine($"item1 time {pitem1.Timestamp} item2 time {pitem2.Timestamp}"); + // pitem1 < pitem2 (fee) => -1 + pitem1.CompareTo(pitem2).Should().Be(-1); + // pitem2 > pitem1 (fee) => 1 + pitem2.CompareTo(pitem1).Should().Be(1); + } + + [TestMethod] + public void PoolItem_CompareTo_Hash() + { + int sizeFixed = 50; + int netFeeSatoshiFixed = 1; + + for (int testRuns = 0; testRuns < 30; testRuns++) + { + var tx1 = GenerateMockTxWithFirstByteOfHashGreaterThanOrEqualTo(0x80, new Fixed8(netFeeSatoshiFixed), sizeFixed); + var tx2 = GenerateMockTxWithFirstByteOfHashLessThanOrEqualTo(0x79,new Fixed8(netFeeSatoshiFixed), sizeFixed); + + PoolItem pitem1 = new PoolItem(tx1.Object); + PoolItem pitem2 = new PoolItem(tx2.Object); + + // pitem2 < pitem1 (fee) => -1 + pitem2.CompareTo(pitem1).Should().Be(-1); + + // pitem1 > pitem2 (fee) => 1 + pitem1.CompareTo(pitem2).Should().Be(1); + } + + // equal hashes should be equal + var tx3 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + var tx4 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + PoolItem pitem3 = new PoolItem(tx3.Object); + PoolItem pitem4 = new PoolItem(tx4.Object); + + pitem3.CompareTo(pitem4).Should().Be(0); + pitem4.CompareTo(pitem3).Should().Be(0); + } + + [TestMethod] + public void PoolItem_CompareTo_Equals() + { + int sizeFixed = 500; + int netFeeSatoshiFixed = 10; + var tx1 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + var tx2 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + + PoolItem pitem1 = new PoolItem(tx1.Object); + PoolItem pitem2 = new PoolItem(tx2.Object); + + // pitem1 == pitem2 (fee) => 0 + pitem1.CompareTo(pitem2).Should().Be(0); + pitem2.CompareTo(pitem1).Should().Be(0); + } + + public Mock GenerateMockTxWithFirstByteOfHashGreaterThanOrEqualTo(byte firstHashByte, Fixed8 networkFee, int size) + { + Mock mockTx; + do + { + mockTx = MockGenerateInvocationTx(networkFee, size); + } while (mockTx.Object.Hash >= new UInt256(TestUtils.GetByteArray(32, firstHashByte))); + + return mockTx; + } + + public Mock GenerateMockTxWithFirstByteOfHashLessThanOrEqualTo(byte firstHashByte, Fixed8 networkFee, int size) + { + Mock mockTx; + do + { + mockTx = MockGenerateInvocationTx(networkFee, size); + } while (mockTx.Object.Hash <= new UInt256(TestUtils.GetByteArray(32, firstHashByte))); + + return mockTx; + } + + + // Generate Mock InvocationTransaction with different sizes and prices + public static Mock MockGenerateInvocationTx(Fixed8 networkFee, int size, byte[] overrideScriptBytes=null) + { + var mockTx = new Mock(); + mockTx.CallBase = true; + mockTx.SetupGet(mr => mr.NetworkFee).Returns(networkFee); + mockTx.SetupGet(mr => mr.Size).Returns(size); + + var tx = mockTx.Object; + // use random bytes in the script to get a different hash since we cannot mock the Hash + byte[] randomBytes; + if (overrideScriptBytes != null) + randomBytes = overrideScriptBytes; + else + { + randomBytes = new byte[16]; + TestRandom.NextBytes(randomBytes); + } + tx.Script = randomBytes; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + return mockTx; + } + } +} diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index c6df72891a..0d857495de 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -14,38 +14,6 @@ namespace Neo.Ledger { public class MemoryPool : IReadOnlyCollection { - private class PoolItem : IComparable - { - public readonly Transaction Tx; - public readonly DateTime Timestamp; - public DateTime LastBroadcastTimestamp; - - public PoolItem(Transaction tx) - { - Tx = tx; - Timestamp = DateTime.UtcNow; - LastBroadcastTimestamp = Timestamp; - } - - public int CompareTo(Transaction otherTx) - { - if (otherTx == null) return 1; - // Fees sorted ascending - int ret = Tx.FeePerByte.CompareTo(otherTx.FeePerByte); - if (ret != 0) return ret; - ret = Tx.NetworkFee.CompareTo(otherTx.NetworkFee); - if (ret != 0) return ret; - // Transaction hash sorted descending - return otherTx.Hash.CompareTo(Tx.Hash); - } - - public int CompareTo(PoolItem otherItem) - { - if (otherItem == null) return 1; - return CompareTo(otherItem.Tx); - } - } - // Allow a reverified transaction to be rebroadcasted if it has been this many block times since last broadcast. private const int BlocksTillRebroadcastLowPriorityPoolTx = 30; private const int BlocksTillRebroadcastHighPriorityPoolTx = 10; diff --git a/neo/Ledger/PoolItem.cs b/neo/Ledger/PoolItem.cs new file mode 100644 index 0000000000..13dc9b9344 --- /dev/null +++ b/neo/Ledger/PoolItem.cs @@ -0,0 +1,63 @@ +using Neo.Network.P2P.Payloads; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using Akka.Util.Internal; +using Neo.Network.P2P; +using Neo.Persistence; +using Neo.Plugins; + +namespace Neo.Ledger +{ + /// + /// Represents an item in the Memory Pool. + /// + // Note: PoolItem objects don't consider transaction priority (low or high) in their compare CompareTo method. + /// This is because items of differing priority are never added to the same sorted set in MemoryPool. + /// + internal class PoolItem : IComparable + { + /// + /// Internal transaction for PoolItem + /// + public readonly Transaction Tx; + + /// + /// Timestamp when transaction was stored on PoolItem + /// + public readonly DateTime Timestamp; + + /// + /// Timestamp where this transaction was last broadcast to other nodes + /// + public DateTime LastBroadcastTimestamp; + + internal PoolItem(Transaction tx) + { + Tx = tx; + Timestamp = TimeProvider.Current.UtcNow; + LastBroadcastTimestamp = Timestamp; + } + + public int CompareTo(Transaction otherTx) + { + if (otherTx == null) return 1; + // Fees sorted ascending + int ret = Tx.FeePerByte.CompareTo(otherTx.FeePerByte); + if (ret != 0) return ret; + ret = Tx.NetworkFee.CompareTo(otherTx.NetworkFee); + if (ret != 0) return ret; + // Transaction hash sorted descending + return otherTx.Hash.CompareTo(Tx.Hash); + } + + public int CompareTo(PoolItem otherItem) + { + if (otherItem == null) return 1; + return CompareTo(otherItem.Tx); + } + } +} From 7f6e552abdea086e170d2f7deb4e393c1d20056c Mon Sep 17 00:00:00 2001 From: jsolman Date: Sat, 19 Jan 2019 19:26:55 -0800 Subject: [PATCH 21/27] Treat Claim transactions as the highest low priority transactions. (#565) --- neo.UnitTests/UT_PoolItem.cs | 26 ++++++++++++++++++++++++++ neo/Ledger/PoolItem.cs | 25 ++++++++++++++----------- 2 files changed, 40 insertions(+), 11 deletions(-) diff --git a/neo.UnitTests/UT_PoolItem.cs b/neo.UnitTests/UT_PoolItem.cs index 3ef28356dc..0d8a8bc19a 100644 --- a/neo.UnitTests/UT_PoolItem.cs +++ b/neo.UnitTests/UT_PoolItem.cs @@ -33,6 +33,19 @@ public void TestCleanup() TimeProvider.ResetToDefault(); } + [TestMethod] + public void PoolItem_CompareTo_ClaimTx() + { + var tx1 = GenerateClaimTx(); + // Non-free low-priority transaction + var tx2 = MockGenerateInvocationTx(new Fixed8(99999), 50).Object; + + var poolItem1 = new PoolItem(tx1); + var poolItem2 = new PoolItem(tx2); + poolItem1.CompareTo(poolItem2).Should().Be(1); + poolItem2.CompareTo(poolItem1).Should().Be(-1); + } + [TestMethod] public void PoolItem_CompareTo_Fee() { @@ -122,6 +135,19 @@ public Mock GenerateMockTxWithFirstByteOfHashLessThanOrEq return mockTx; } + public static Transaction GenerateClaimTx() + { + var mockTx = new Mock(); + mockTx.CallBase = true; + mockTx.SetupGet(mr => mr.NetworkFee).Returns(Fixed8.Zero); + mockTx.SetupGet(mr => mr.Size).Returns(50); + var tx = mockTx.Object; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + return mockTx.Object; + } // Generate Mock InvocationTransaction with different sizes and prices public static Mock MockGenerateInvocationTx(Fixed8 networkFee, int size, byte[] overrideScriptBytes=null) diff --git a/neo/Ledger/PoolItem.cs b/neo/Ledger/PoolItem.cs index 13dc9b9344..3810330dc4 100644 --- a/neo/Ledger/PoolItem.cs +++ b/neo/Ledger/PoolItem.cs @@ -1,19 +1,10 @@ using Neo.Network.P2P.Payloads; using System; -using System.Collections; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Threading; -using Akka.Util.Internal; -using Neo.Network.P2P; -using Neo.Persistence; -using Neo.Plugins; namespace Neo.Ledger { /// - /// Represents an item in the Memory Pool. + /// Represents an item in the Memory Pool. /// // Note: PoolItem objects don't consider transaction priority (low or high) in their compare CompareTo method. /// This is because items of differing priority are never added to the same sorted set in MemoryPool. @@ -31,7 +22,7 @@ internal class PoolItem : IComparable public readonly DateTime Timestamp; /// - /// Timestamp where this transaction was last broadcast to other nodes + /// Timestamp when this transaction was last broadcast to other nodes /// public DateTime LastBroadcastTimestamp; @@ -45,6 +36,18 @@ internal PoolItem(Transaction tx) public int CompareTo(Transaction otherTx) { if (otherTx == null) return 1; + if (Tx.IsLowPriority && otherTx.IsLowPriority) + { + bool thisIsClaimTx = Tx is ClaimTransaction; + bool otherIsClaimTx = otherTx is ClaimTransaction; + if (thisIsClaimTx != otherIsClaimTx) + { + // This is a claim Tx and other isn't. + if (thisIsClaimTx) return 1; + // The other is claim Tx and this isn't. + return -1; + } + } // Fees sorted ascending int ret = Tx.FeePerByte.CompareTo(otherTx.FeePerByte); if (ret != 0) return ret; From bbbc0cf5e401e425e7d749f26b23b824953da10a Mon Sep 17 00:00:00 2001 From: jsolman Date: Wed, 23 Jan 2019 14:07:02 -0800 Subject: [PATCH 22/27] Allow persistence plugins to commit as a final step. (#568) * Allow persistence plugins to commit as a final step. * Plugins commit before core commits, once all plugins have handled initial work OnPersist. * Allow PersistencePlugin to determine whether to crash if commit fails. * Add ShouldThrowExceptionFromCommit method to IPersistencePlugin. * Throw all commit exceptions that should be thrown in an AggregateException. --- neo/Ledger/Blockchain.cs | 19 +++++++++++++++++++ neo/Plugins/IPersistencePlugin.cs | 5 ++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/neo/Ledger/Blockchain.cs b/neo/Ledger/Blockchain.cs index fa9d7dc5fb..82dac883a6 100644 --- a/neo/Ledger/Blockchain.cs +++ b/neo/Ledger/Blockchain.cs @@ -610,6 +610,25 @@ private void Persist(Block block) foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins) plugin.OnPersist(snapshot, all_application_executed); snapshot.Commit(); + List commitExceptions = null; + foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins) + { + try + { + plugin.OnCommit(snapshot); + } + catch (Exception ex) + { + if (plugin.ShouldThrowExceptionFromCommit(ex)) + { + if (commitExceptions == null) + commitExceptions = new List(); + + commitExceptions.Add(ex); + } + } + } + if (commitExceptions != null) throw new AggregateException(commitExceptions); } UpdateCurrentSnapshot(); OnPersistCompleted(block); diff --git a/neo/Plugins/IPersistencePlugin.cs b/neo/Plugins/IPersistencePlugin.cs index cf4ee697fd..af3cbe05b8 100644 --- a/neo/Plugins/IPersistencePlugin.cs +++ b/neo/Plugins/IPersistencePlugin.cs @@ -1,4 +1,5 @@ -using Neo.Persistence; +using System; +using Neo.Persistence; using System.Collections.Generic; using static Neo.Ledger.Blockchain; @@ -7,5 +8,7 @@ namespace Neo.Plugins public interface IPersistencePlugin { void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList); + void OnCommit(Snapshot snapshot); + bool ShouldThrowExceptionFromCommit(Exception ex); } } From 7cf286941f7f5541e2ab4bf182583452eec8fc5a Mon Sep 17 00:00:00 2001 From: jsolman Date: Sun, 10 Feb 2019 17:14:52 -0800 Subject: [PATCH 23/27] Add a Plugin type for observing transactions added or removed from the MemoryPool. (#580) --- neo/Ledger/MemoryPool.cs | 30 +++++++++++++++++----- neo/Plugins/IMemoryPoolTxObserverPlugin.cs | 11 ++++++++ neo/Plugins/MemoryPoolTxRemovalReason.cs | 14 ++++++++++ neo/Plugins/Plugin.cs | 2 ++ 4 files changed, 51 insertions(+), 6 deletions(-) create mode 100644 neo/Plugins/IMemoryPoolTxObserverPlugin.cs create mode 100644 neo/Plugins/MemoryPoolTxRemovalReason.cs diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 0d857495de..edeb66f0f2 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -273,9 +273,10 @@ internal bool CanTransactionFitInPool(Transaction tx) } /// + /// Adds an already verified transaction to the memory pool. /// - /// Note: This must only be called from a single thread (the Blockchain actor) to add a transaction to the pool - /// one should tell the Blockchain actor about the transaction. + /// Note: This must only be called from a single thread (the Blockchain actor). To add a transaction to the pool + /// tell the Blockchain actor about the transaction. /// /// /// @@ -286,6 +287,7 @@ internal bool TryAdd(UInt256 hash, Transaction tx) if (_unsortedTransactions.ContainsKey(hash)) return false; + List removedTransactions = null; _txRwLock.EnterWriteLock(); try { @@ -293,25 +295,37 @@ internal bool TryAdd(UInt256 hash, Transaction tx) SortedSet pool = tx.IsLowPriority ? _sortedLowPrioTransactions : _sortedHighPrioTransactions; pool.Add(poolItem); - RemoveOverCapacity(); + if (Count > Capacity) + removedTransactions = RemoveOverCapacity(); } finally { _txRwLock.ExitWriteLock(); } + foreach (IMemoryPoolTxObserverPlugin plugin in Plugin.TxObserverPlugins) + { + plugin.TransactionAdded(poolItem.Tx); + if (removedTransactions != null) + plugin.TransactionsRemoved(MemoryPoolTxRemovalReason.CapacityExceeded, removedTransactions); + } + return _unsortedTransactions.ContainsKey(hash); } - private void RemoveOverCapacity() + private List RemoveOverCapacity() { - while (Count > Capacity) + List removedTransactions = new List(); + do { PoolItem minItem = GetLowestFeeTransaction(out var unsortedPool, out var sortedPool); unsortedPool.Remove(minItem.Tx.Hash); sortedPool.Remove(minItem); - } + removedTransactions.Add(minItem.Tx); + } while (Count > Capacity); + + return removedTransactions; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -450,6 +464,10 @@ internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) _txRwLock.ExitWriteLock(); } + var invalidTransactions = invalidItems.Select(p => p.Tx).ToArray(); + foreach (IMemoryPoolTxObserverPlugin plugin in Plugin.TxObserverPlugins) + plugin.TransactionsRemoved(MemoryPoolTxRemovalReason.NoLongerValid, invalidTransactions); + return reverifiedItems.Count; } diff --git a/neo/Plugins/IMemoryPoolTxObserverPlugin.cs b/neo/Plugins/IMemoryPoolTxObserverPlugin.cs new file mode 100644 index 0000000000..e596b9a7b5 --- /dev/null +++ b/neo/Plugins/IMemoryPoolTxObserverPlugin.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using Neo.Network.P2P.Payloads; + +namespace Neo.Plugins +{ + public interface IMemoryPoolTxObserverPlugin + { + void TransactionAdded(Transaction tx); + void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable transactions); + } +} diff --git a/neo/Plugins/MemoryPoolTxRemovalReason.cs b/neo/Plugins/MemoryPoolTxRemovalReason.cs new file mode 100644 index 0000000000..f7320dc60e --- /dev/null +++ b/neo/Plugins/MemoryPoolTxRemovalReason.cs @@ -0,0 +1,14 @@ +namespace Neo.Plugins +{ + public enum MemoryPoolTxRemovalReason : byte + { + /// + /// The transaction was ejected since it was the lowest priority transaction and the MemoryPool capacity was exceeded. + /// + CapacityExceeded, + /// + /// The transaction was ejected due to failing re-validation after a block was persisted. + /// + NoLongerValid, + } +} \ No newline at end of file diff --git a/neo/Plugins/Plugin.cs b/neo/Plugins/Plugin.cs index 58a9bb006d..3f32fcb14c 100644 --- a/neo/Plugins/Plugin.cs +++ b/neo/Plugins/Plugin.cs @@ -16,6 +16,7 @@ public abstract class Plugin internal static readonly List Policies = new List(); internal static readonly List RpcPlugins = new List(); internal static readonly List PersistencePlugins = new List(); + internal static readonly List TxObserverPlugins = new List(); private static readonly string pluginsPath = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), "Plugins"); private static readonly FileSystemWatcher configWatcher; @@ -51,6 +52,7 @@ protected Plugin() if (this is IPolicyPlugin policy) Policies.Add(policy); if (this is IRpcPlugin rpc) RpcPlugins.Add(rpc); if (this is IPersistencePlugin persistence) PersistencePlugins.Add(persistence); + if (this is IMemoryPoolTxObserverPlugin txObserver) TxObserverPlugins.Add(txObserver); Configure(); } From 6ae3c6d8324663ac77d747a0e71b03a0617b6ece Mon Sep 17 00:00:00 2001 From: Erik Zhang Date: Mon, 18 Feb 2019 04:10:16 +0800 Subject: [PATCH 24/27] Correctly handle conversions between JSON objects (#586) --- .travis.yml | 2 +- neo/IO/Json/JArray.cs | 6 +++ neo/IO/Json/JBoolean.cs | 14 ++---- neo/IO/Json/JNumber.cs | 64 ++++++++------------------ neo/IO/Json/JObject.cs | 49 ++++---------------- neo/IO/Json/JString.cs | 61 ++++++------------------ neo/Network/RPC/RpcServer.cs | 8 ++-- neo/SmartContract/ContractParameter.cs | 2 +- neo/Wallets/NEP6/NEP6Contract.cs | 2 +- neo/neo.csproj | 1 + 10 files changed, 62 insertions(+), 147 deletions(-) diff --git a/.travis.yml b/.travis.yml index 772858f229..41d54f6fc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ dist: trusty osx_image: xcode9.1 mono: none -dotnet: 2.0.0 +dotnet: 2.1.502 before_install: - cd neo.UnitTests diff --git a/neo/IO/Json/JArray.cs b/neo/IO/Json/JArray.cs index 0fc758a0bf..02911d505f 100644 --- a/neo/IO/Json/JArray.cs +++ b/neo/IO/Json/JArray.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; namespace Neo.IO.Json @@ -52,6 +53,11 @@ public void Add(JObject item) items.Add(item); } + public override string AsString() + { + return string.Join(",", items.Select(p => p?.AsString())); + } + public void Clear() { items.Clear(); diff --git a/neo/IO/Json/JBoolean.cs b/neo/IO/Json/JBoolean.cs index bf941fc860..6cc39ed231 100644 --- a/neo/IO/Json/JBoolean.cs +++ b/neo/IO/Json/JBoolean.cs @@ -17,18 +17,14 @@ public override bool AsBoolean() return Value; } - public override string AsString() + public override double AsNumber() { - return Value.ToString().ToLower(); + return Value ? 1 : 0; } - public override bool CanConvertTo(Type type) + public override string AsString() { - if (type == typeof(bool)) - return true; - if (type == typeof(string)) - return true; - return false; + return Value.ToString().ToLower(); } internal static JBoolean Parse(TextReader reader) @@ -61,7 +57,7 @@ internal static JBoolean Parse(TextReader reader) public override string ToString() { - return Value.ToString().ToLower(); + return AsString(); } } } diff --git a/neo/IO/Json/JNumber.cs b/neo/IO/Json/JNumber.cs index bf9451975a..fdeb495211 100644 --- a/neo/IO/Json/JNumber.cs +++ b/neo/IO/Json/JNumber.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Reflection; using System.Text; namespace Neo.IO.Json @@ -16,34 +15,7 @@ public JNumber(double value = 0) public override bool AsBoolean() { - if (Value == 0) - return false; - return true; - } - - public override T AsEnum(bool ignoreCase = false) - { - Type t = typeof(T); - TypeInfo ti = t.GetTypeInfo(); - if (!ti.IsEnum) - throw new InvalidCastException(); - if (ti.GetEnumUnderlyingType() == typeof(byte)) - return (T)Enum.ToObject(t, (byte)Value); - if (ti.GetEnumUnderlyingType() == typeof(int)) - return (T)Enum.ToObject(t, (int)Value); - if (ti.GetEnumUnderlyingType() == typeof(long)) - return (T)Enum.ToObject(t, (long)Value); - if (ti.GetEnumUnderlyingType() == typeof(sbyte)) - return (T)Enum.ToObject(t, (sbyte)Value); - if (ti.GetEnumUnderlyingType() == typeof(short)) - return (T)Enum.ToObject(t, (short)Value); - if (ti.GetEnumUnderlyingType() == typeof(uint)) - return (T)Enum.ToObject(t, (uint)Value); - if (ti.GetEnumUnderlyingType() == typeof(ulong)) - return (T)Enum.ToObject(t, (ulong)Value); - if (ti.GetEnumUnderlyingType() == typeof(ushort)) - return (T)Enum.ToObject(t, (ushort)Value); - throw new InvalidCastException(); + return Value != 0 && !double.IsNaN(Value); } public override double AsNumber() @@ -53,23 +25,11 @@ public override double AsNumber() public override string AsString() { + if (double.IsPositiveInfinity(Value)) return "Infinity"; + if (double.IsNegativeInfinity(Value)) return "-Infinity"; return Value.ToString(); } - public override bool CanConvertTo(Type type) - { - if (type == typeof(bool)) - return true; - if (type == typeof(double)) - return true; - if (type == typeof(string)) - return true; - TypeInfo ti = type.GetTypeInfo(); - if (ti.IsEnum && Enum.IsDefined(type, Convert.ChangeType(Value, ti.GetEnumUnderlyingType()))) - return true; - return false; - } - internal static JNumber Parse(TextReader reader) { SkipSpace(reader); @@ -92,7 +52,7 @@ internal static JNumber Parse(TextReader reader) public override string ToString() { - return Value.ToString(); + return AsString(); } public DateTime ToTimestamp() @@ -101,5 +61,21 @@ public DateTime ToTimestamp() throw new InvalidCastException(); return ((ulong)Value).ToDateTime(); } + + public override T TryGetEnum(T defaultValue = default, bool ignoreCase = false) + { + Type enumType = typeof(T); + object value; + try + { + value = Convert.ChangeType(Value, enumType.GetEnumUnderlyingType()); + } + catch (OverflowException) + { + return defaultValue; + } + object result = Enum.ToObject(enumType, value); + return Enum.IsDefined(enumType, result) ? (T)result : defaultValue; + } } } diff --git a/neo/IO/Json/JObject.cs b/neo/IO/Json/JObject.cs index 3d7d9df7b0..a477e49902 100644 --- a/neo/IO/Json/JObject.cs +++ b/neo/IO/Json/JObject.cs @@ -27,55 +27,17 @@ public class JObject public virtual bool AsBoolean() { - throw new InvalidCastException(); - } - - public bool AsBooleanOrDefault(bool value = false) - { - if (!CanConvertTo(typeof(bool))) - return value; - return AsBoolean(); - } - - public virtual T AsEnum(bool ignoreCase = false) - { - throw new InvalidCastException(); - } - - public T AsEnumOrDefault(T value = default(T), bool ignoreCase = false) - { - if (!CanConvertTo(typeof(T))) - return value; - return AsEnum(ignoreCase); + return true; } public virtual double AsNumber() { - throw new InvalidCastException(); - } - - public double AsNumberOrDefault(double value = 0) - { - if (!CanConvertTo(typeof(double))) - return value; - return AsNumber(); + return double.NaN; } public virtual string AsString() { - throw new InvalidCastException(); - } - - public string AsStringOrDefault(string value = null) - { - if (!CanConvertTo(typeof(string))) - return value; - return AsString(); - } - - public virtual bool CanConvertTo(Type type) - { - return false; + return "[object Object]"; } public bool ContainsProperty(string key) @@ -189,6 +151,11 @@ public override string ToString() return sb.ToString(); } + public virtual T TryGetEnum(T defaultValue = default, bool ignoreCase = false) where T : Enum + { + return defaultValue; + } + public static implicit operator JObject(Enum value) { return new JString(value.ToString()); diff --git a/neo/IO/Json/JString.cs b/neo/IO/Json/JString.cs index c9fac5a9a2..f208ea95c5 100644 --- a/neo/IO/Json/JString.cs +++ b/neo/IO/Json/JString.cs @@ -1,7 +1,6 @@ using System; using System.Globalization; using System.IO; -using System.Reflection; using System.Text; using System.Text.Encodings.Web; @@ -18,42 +17,13 @@ public JString(string value) public override bool AsBoolean() { - switch (Value.ToLower()) - { - case "0": - case "f": - case "false": - case "n": - case "no": - case "off": - return false; - default: - return true; - } - } - - public override T AsEnum(bool ignoreCase = false) - { - try - { - return (T)Enum.Parse(typeof(T), Value, ignoreCase); - } - catch - { - throw new InvalidCastException(); - } + return !string.IsNullOrEmpty(Value); } public override double AsNumber() { - try - { - return double.Parse(Value); - } - catch - { - throw new InvalidCastException(); - } + if (string.IsNullOrEmpty(Value)) return 0; + return double.TryParse(Value, out double result) ? result : double.NaN; } public override string AsString() @@ -61,19 +31,6 @@ public override string AsString() return Value; } - public override bool CanConvertTo(Type type) - { - if (type == typeof(bool)) - return true; - if (type.GetTypeInfo().IsEnum && Enum.IsDefined(type, Value)) - return true; - if (type == typeof(double)) - return true; - if (type == typeof(string)) - return true; - return false; - } - internal static JString Parse(TextReader reader) { SkipSpace(reader); @@ -112,5 +69,17 @@ public override string ToString() { return $"\"{JavaScriptEncoder.Default.Encode(Value)}\""; } + + public override T TryGetEnum(T defaultValue = default, bool ignoreCase = false) + { + try + { + return (T)Enum.Parse(typeof(T), Value, ignoreCase); + } + catch + { + return defaultValue; + } + } } } diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index 5a12f190ba..9ba441c381 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -200,7 +200,7 @@ private JObject Process(string method, JArray _params) } if (block == null) throw new RpcException(-100, "Unknown block"); - bool verbose = _params.Count >= 2 && _params[1].AsBooleanOrDefault(false); + bool verbose = _params.Count >= 2 && _params[1].AsBoolean(); if (verbose) { JObject json = block.ToJson(); @@ -239,7 +239,7 @@ private JObject Process(string method, JArray _params) if (header == null) throw new RpcException(-100, "Unknown block"); - bool verbose = _params.Count >= 2 && _params[1].AsBooleanOrDefault(false); + bool verbose = _params.Count >= 2 && _params[1].AsBoolean(); if (verbose) { JObject json = header.ToJson(); @@ -301,7 +301,7 @@ private JObject Process(string method, JArray _params) } case "getrawmempool": { - bool shouldGetUnverified = _params.Count >= 1 && _params[0].AsBooleanOrDefault(false); + bool shouldGetUnverified = _params.Count >= 1 && _params[0].AsBoolean(); if (!shouldGetUnverified) return new JArray(Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => (JObject)p.Hash.ToString())); @@ -317,7 +317,7 @@ private JObject Process(string method, JArray _params) case "getrawtransaction": { UInt256 hash = UInt256.Parse(_params[0].AsString()); - bool verbose = _params.Count >= 2 && _params[1].AsBooleanOrDefault(false); + bool verbose = _params.Count >= 2 && _params[1].AsBoolean(); Transaction tx = Blockchain.Singleton.GetTransaction(hash); if (tx == null) throw new RpcException(-100, "Unknown transaction"); diff --git a/neo/SmartContract/ContractParameter.cs b/neo/SmartContract/ContractParameter.cs index 0b0f76703e..5e78005e13 100644 --- a/neo/SmartContract/ContractParameter.cs +++ b/neo/SmartContract/ContractParameter.cs @@ -59,7 +59,7 @@ public static ContractParameter FromJson(JObject json) { ContractParameter parameter = new ContractParameter { - Type = json["type"].AsEnum() + Type = json["type"].TryGetEnum() }; if (json["value"] != null) switch (parameter.Type) diff --git a/neo/Wallets/NEP6/NEP6Contract.cs b/neo/Wallets/NEP6/NEP6Contract.cs index d934cf0326..a3dd36f414 100644 --- a/neo/Wallets/NEP6/NEP6Contract.cs +++ b/neo/Wallets/NEP6/NEP6Contract.cs @@ -15,7 +15,7 @@ public static NEP6Contract FromJson(JObject json) return new NEP6Contract { Script = json["script"].AsString().HexToBytes(), - ParameterList = ((JArray)json["parameters"]).Select(p => p["type"].AsEnum()).ToArray(), + ParameterList = ((JArray)json["parameters"]).Select(p => p["type"].TryGetEnum()).ToArray(), ParameterNames = ((JArray)json["parameters"]).Select(p => p["name"].AsString()).ToArray(), Deployed = json["deployed"].AsBoolean() }; diff --git a/neo/neo.csproj b/neo/neo.csproj index f922aae876..c9752a83fd 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -16,6 +16,7 @@ Neo The Neo Project Neo + latest From 48890b6ba9acf2349d2a6704816b1d89fc7581ae Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Mon, 18 Feb 2019 07:07:58 +0300 Subject: [PATCH 25/27] Fix neo-project/neo-cli#297 (#587) --- neo/Network/RPC/RpcServer.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index 9ba441c381..1dcce9f107 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -669,6 +669,14 @@ private JObject ProcessRequest(HttpContext context, JObject request) if (result == null) result = Process(method, _params); } + catch (FormatException) + { + return CreateErrorResponse(request["id"], -32602, "Invalid params"); + } + catch (IndexOutOfRangeException) + { + return CreateErrorResponse(request["id"], -32602, "Invalid params"); + } catch (Exception ex) { #if DEBUG From d9fc9e934fd51359d86e7f644153abbaaa0741d2 Mon Sep 17 00:00:00 2001 From: Evgeniy Kulikov Date: Tue, 19 Feb 2019 10:36:53 +0300 Subject: [PATCH 26/27] Replace new JArray with .ToArray (AccountState) (#581) --- neo/Ledger/AccountState.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/neo/Ledger/AccountState.cs b/neo/Ledger/AccountState.cs index d34738cfb7..b53bcc8e8d 100644 --- a/neo/Ledger/AccountState.cs +++ b/neo/Ledger/AccountState.cs @@ -92,14 +92,14 @@ public override JObject ToJson() JObject json = base.ToJson(); json["script_hash"] = ScriptHash.ToString(); json["frozen"] = IsFrozen; - json["votes"] = new JArray(Votes.Select(p => (JObject)p.ToString())); - json["balances"] = new JArray(Balances.Select(p => + json["votes"] = Votes.Select(p => (JObject)p.ToString()).ToArray(); + json["balances"] = Balances.Select(p => { JObject balance = new JObject(); balance["asset"] = p.Key.ToString(); balance["value"] = p.Value.ToString(); return balance; - })); + }).ToArray(); return json; } } From 2bff318d1d1c8c1d055f111ef08eb5743124a6fa Mon Sep 17 00:00:00 2001 From: erikzhang Date: Wed, 20 Feb 2019 17:14:46 +0800 Subject: [PATCH 27/27] Ensure `LocalNode` to be stoped before shutting down the `NeoSystem` --- neo/NeoSystem.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/neo/NeoSystem.cs b/neo/NeoSystem.cs index ea0c1a6cdf..efeb8e6f3e 100644 --- a/neo/NeoSystem.cs +++ b/neo/NeoSystem.cs @@ -8,6 +8,7 @@ using Neo.Wallets; using System; using System.Net; +using System.Threading; namespace Neo { @@ -40,10 +41,18 @@ public NeoSystem(Store store) public void Dispose() { RpcServer?.Dispose(); - ActorSystem.Stop(LocalNode); + EnsureStoped(LocalNode); ActorSystem.Dispose(); } + public void EnsureStoped(IActorRef actor) + { + Inbox inbox = Inbox.Create(ActorSystem); + inbox.Watch(actor); + ActorSystem.Stop(actor); + inbox.Receive(Timeout.InfiniteTimeSpan); + } + internal void ResumeNodeStartup() { suspend = false;