diff --git a/.travis.yml b/.travis.yml index 772858f229..41d54f6fc4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ dist: trusty osx_image: xcode9.1 mono: none -dotnet: 2.0.0 +dotnet: 2.1.502 before_install: - cd neo.UnitTests diff --git a/neo.UnitTests/TestDataCache.cs b/neo.UnitTests/TestDataCache.cs index d2d3afdabf..68f3b07da7 100644 --- a/neo.UnitTests/TestDataCache.cs +++ b/neo.UnitTests/TestDataCache.cs @@ -10,6 +10,17 @@ public class TestDataCache : DataCache where TKey : IEquatable, ISerializable where TValue : class, ICloneable, ISerializable, new() { + private readonly TValue _defaultValue; + + public TestDataCache() + { + _defaultValue = null; + } + + public TestDataCache(TValue defaultValue) + { + this._defaultValue = defaultValue; + } public override void DeleteInternal(TKey key) { } @@ -25,12 +36,13 @@ protected override void AddInternal(TKey key, TValue value) protected override TValue GetInternal(TKey key) { - throw new NotImplementedException(); + if (_defaultValue == null) throw new NotImplementedException(); + return _defaultValue; } protected override TValue TryGetInternal(TKey key) { - return null; + return _defaultValue; } protected override void UpdateInternal(TKey key, TValue value) diff --git a/neo.UnitTests/UT_MemoryPool.cs b/neo.UnitTests/UT_MemoryPool.cs new file mode 100644 index 0000000000..4f78b9b033 --- /dev/null +++ b/neo.UnitTests/UT_MemoryPool.cs @@ -0,0 +1,401 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Ledger; +using FluentAssertions; +using Neo.Cryptography.ECC; +using Neo.IO.Wrappers; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_MemoryPool + { + private static NeoSystem TheNeoSystem; + + private readonly Random _random = new Random(1337); // use fixed seed for guaranteed determinism + + private MemoryPool _unit; + + [TestInitialize] + public void TestSetup() + { + // protect against external changes on TimeProvider + TimeProvider.ResetToDefault(); + + if (TheNeoSystem == null) + { + var mockSnapshot = new Mock(); + mockSnapshot.SetupGet(p => p.Blocks).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Transactions).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Accounts).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.UnspentCoins).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.SpentCoins).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Validators).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Assets).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Contracts).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.Storages).Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.HeaderHashList) + .Returns(new TestDataCache()); + mockSnapshot.SetupGet(p => p.ValidatorsCount).Returns(new TestMetaDataCache()); + mockSnapshot.SetupGet(p => p.BlockHashIndex).Returns(new TestMetaDataCache()); + mockSnapshot.SetupGet(p => p.HeaderHashIndex).Returns(new TestMetaDataCache()); + + var mockStore = new Mock(); + + var defaultTx = CreateRandomHashInvocationMockTransaction().Object; + defaultTx.Outputs = new TransactionOutput[1]; + defaultTx.Outputs[0] = new TransactionOutput + { + AssetId = Blockchain.UtilityToken.Hash, + Value = new Fixed8(1000000), + ScriptHash = UInt160.Zero // doesn't matter for our purposes. + }; + + mockStore.Setup(p => p.GetBlocks()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetTransactions()).Returns(new TestDataCache( + new TransactionState + { + BlockIndex = 1, + Transaction = defaultTx + })); + + mockStore.Setup(p => p.GetAccounts()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetUnspentCoins()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetSpentCoins()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetValidators()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetAssets()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetContracts()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetStorages()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetHeaderHashList()).Returns(new TestDataCache()); + mockStore.Setup(p => p.GetValidatorsCount()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetBlockHashIndex()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetHeaderHashIndex()).Returns(new TestMetaDataCache()); + mockStore.Setup(p => p.GetSnapshot()).Returns(mockSnapshot.Object); + + Console.WriteLine("initialize NeoSystem"); + TheNeoSystem = new NeoSystem(mockStore.Object); // new Mock(mockStore.Object); + } + + // Create a MemoryPool with capacity of 100 + _unit = new MemoryPool(TheNeoSystem, 100); + + // Verify capacity equals the amount specified + _unit.Capacity.ShouldBeEquivalentTo(100); + + _unit.VerifiedCount.ShouldBeEquivalentTo(0); + _unit.UnVerifiedCount.ShouldBeEquivalentTo(0); + _unit.Count.ShouldBeEquivalentTo(0); + } + + private Mock CreateRandomHashInvocationMockTransaction() + { + var mockTx = new Mock(); + mockTx.CallBase = true; + mockTx.Setup(p => p.Verify(It.IsAny(), It.IsAny>())).Returns(true); + var tx = mockTx.Object; + var randomBytes = new byte[16]; + _random.NextBytes(randomBytes); + tx.Script = randomBytes; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + + return mockTx; + } + + long LongRandom(long min, long max, Random rand) + { + // Only returns positive random long values. + long longRand = (long) rand.NextBigInteger(63); + return longRand % (max - min) + min; + } + + private Transaction CreateMockTransactionWithFee(long fee) + { + var mockTx = CreateRandomHashInvocationMockTransaction(); + mockTx.SetupGet(p => p.NetworkFee).Returns(new Fixed8(fee)); + var tx = mockTx.Object; + if (fee > 0) + { + tx.Inputs = new CoinReference[1]; + // Any input will trigger reading the transaction output and get our mocked transaction output. + tx.Inputs[0] = new CoinReference + { + PrevHash = UInt256.Zero, + PrevIndex = 0 + }; + } + return tx; + } + + private Transaction CreateMockHighPriorityTransaction() + { + return CreateMockTransactionWithFee(LongRandom(100000, 100000000, _random)); + } + + private Transaction CreateMockLowPriorityTransaction() + { + long rNetFee = LongRandom(0, 100000, _random); + // [0,0.001] GAS a fee lower than the threshold of 0.001 GAS (not enough to be a high priority TX) + return CreateMockTransactionWithFee(rNetFee); + } + + private void AddTransactions(int count, bool isHighPriority=false) + { + for (int i = 0; i < count; i++) + { + var txToAdd = isHighPriority ? CreateMockHighPriorityTransaction(): CreateMockLowPriorityTransaction(); + Console.WriteLine($"created tx: {txToAdd.Hash}"); + _unit.TryAdd(txToAdd.Hash, txToAdd); + } + } + + private void AddLowPriorityTransactions(int count) => AddTransactions(count); + public void AddHighPriorityTransactions(int count) => AddTransactions(count, true); + + [TestMethod] + public void LowPriorityCapacityTest() + { + // Add over the capacity items, verify that the verified count increases each time + AddLowPriorityTransactions(50); + _unit.VerifiedCount.ShouldBeEquivalentTo(50); + AddLowPriorityTransactions(51); + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}"); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(100); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + + _unit.VerifiedCount.ShouldBeEquivalentTo(100); + _unit.UnVerifiedCount.ShouldBeEquivalentTo(0); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void HighPriorityCapacityTest() + { + // Add over the capacity items, verify that the verified count increases each time + AddHighPriorityTransactions(101); + + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}"); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(100); + + _unit.VerifiedCount.ShouldBeEquivalentTo(100); + _unit.UnVerifiedCount.ShouldBeEquivalentTo(0); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void HighPriorityPushesOutLowPriority() + { + // Add over the capacity items, verify that the verified count increases each time + AddLowPriorityTransactions(70); + AddHighPriorityTransactions(40); + + Console.WriteLine($"VerifiedCount: {_unit.VerifiedCount} LowPrioCount {_unit.SortedLowPrioTxCount} HighPrioCount {_unit.SortedHighPrioTxCount}"); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(40); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void LowPriorityDoesNotPushOutHighPrority() + { + AddHighPriorityTransactions(70); + AddLowPriorityTransactions(40); + + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(30); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(70); + _unit.Count.ShouldBeEquivalentTo(100); + } + + [TestMethod] + public void BlockPersistMovesTxToUnverifiedAndReverification() + { + AddHighPriorityTransactions(70); + AddLowPriorityTransactions(30); + + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(70); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(30); + + var block = new Block + { + Transactions = _unit.GetSortedVerifiedTransactions().Take(10) + .Concat(_unit.GetSortedVerifiedTransactions().Where(x => x.IsLowPriority).Take(5)).ToArray() + }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(25); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(9); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(1); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(51); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(24); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(18); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(2); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(42); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(23); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(27); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(3); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(33); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(22); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(36); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(4); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(24); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(21); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(45); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(5); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(15); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(20); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(54); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(6); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(6); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(19); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(10); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(15); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(20); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(5); + + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(10, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(60); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(25); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(0); + } + + private void verifyTransactionsSortedDescending(IEnumerable transactions) + { + Transaction lastTransaction = null; + foreach (var tx in transactions) + { + if (lastTransaction != null) + { + if (lastTransaction.FeePerByte == tx.FeePerByte) + { + if (lastTransaction.NetworkFee == tx.NetworkFee) + lastTransaction.Hash.Should().BeLessThan(tx.Hash); + else + lastTransaction.NetworkFee.Should().BeGreaterThan(tx.NetworkFee); + } + else + { + lastTransaction.FeePerByte.Should().BeGreaterThan(tx.FeePerByte); + } + } + lastTransaction = tx; + } + } + + [TestMethod] + public void VerifySortOrderAndThatHighetFeeTransactionsAreReverifiedFirst() + { + AddLowPriorityTransactions(50); + AddHighPriorityTransactions(50); + + var sortedVerifiedTxs = _unit.GetSortedVerifiedTransactions().ToList(); + // verify all 100 transactions are returned in sorted order + sortedVerifiedTxs.Count.ShouldBeEquivalentTo(100); + verifyTransactionsSortedDescending(sortedVerifiedTxs); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(50); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(50); + + // We can verify the order they are re-verified by reverifying 2 at a time + while (_unit.UnVerifiedCount > 0) + { + _unit.GetVerifiedAndUnverifiedTransactions(out IEnumerable sortedVerifiedTransactions, + out IEnumerable sortedUnverifiedTransactions); + sortedVerifiedTransactions.Count().ShouldBeEquivalentTo(0); + var sortedUnverifiedArray = sortedUnverifiedTransactions.ToArray(); + verifyTransactionsSortedDescending(sortedUnverifiedArray); + var maxHighPriorityTransaction = sortedUnverifiedArray.First(); + var maxLowPriorityTransaction = sortedUnverifiedArray.First(tx => tx.IsLowPriority); + + // reverify 1 high priority and 1 low priority transaction + _unit.ReVerifyTopUnverifiedTransactionsIfNeeded(2, Blockchain.Singleton.GetSnapshot()); + var verifiedTxs = _unit.GetSortedVerifiedTransactions().ToArray(); + verifiedTxs.Length.ShouldBeEquivalentTo(2); + verifiedTxs[0].ShouldBeEquivalentTo(maxHighPriorityTransaction); + verifiedTxs[1].ShouldBeEquivalentTo(maxLowPriorityTransaction); + var blockWith2Tx = new Block { Transactions = new Transaction[2] { maxHighPriorityTransaction, maxLowPriorityTransaction }}; + // verify and remove the 2 transactions from the verified pool + _unit.UpdatePoolForBlockPersisted(blockWith2Tx, Blockchain.Singleton.GetSnapshot()); + _unit.SortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.SortedLowPrioTxCount.ShouldBeEquivalentTo(0); + } + _unit.UnverifiedSortedHighPrioTxCount.ShouldBeEquivalentTo(0); + _unit.UnverifiedSortedLowPrioTxCount.ShouldBeEquivalentTo(0); + } + + void VerifyCapacityThresholdForAttemptingToAddATransaction() + { + var sortedVerified = _unit.GetSortedVerifiedTransactions().ToArray(); + + var txBarelyWontFit = CreateMockTransactionWithFee(sortedVerified.Last().NetworkFee.GetData() - 1); + _unit.CanTransactionFitInPool(txBarelyWontFit).ShouldBeEquivalentTo(false); + var txBarelyFits = CreateMockTransactionWithFee(sortedVerified.Last().NetworkFee.GetData() + 1); + _unit.CanTransactionFitInPool(txBarelyFits).ShouldBeEquivalentTo(true); + } + + [TestMethod] + public void VerifyCanTransactionFitInPoolWorksAsIntended() + { + AddLowPriorityTransactions(100); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddHighPriorityTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + AddHighPriorityTransactions(50); + VerifyCapacityThresholdForAttemptingToAddATransaction(); + } + + [TestMethod] + public void CapacityTestWithUnverifiedHighProirtyTransactions() + { + // Verify that unverified high priority transactions will not be pushed out of the queue by incoming + // low priority transactions + + // Fill pool with high priority transactions + AddHighPriorityTransactions(99); + + // move all to unverified + var block = new Block { Transactions = new Transaction[0] }; + _unit.UpdatePoolForBlockPersisted(block, Blockchain.Singleton.GetSnapshot()); + + _unit.CanTransactionFitInPool(CreateMockLowPriorityTransaction()).ShouldBeEquivalentTo(true); + AddHighPriorityTransactions(1); + _unit.CanTransactionFitInPool(CreateMockLowPriorityTransaction()).ShouldBeEquivalentTo(false); + } + } +} diff --git a/neo.UnitTests/UT_PoolItem.cs b/neo.UnitTests/UT_PoolItem.cs new file mode 100644 index 0000000000..0d8a8bc19a --- /dev/null +++ b/neo.UnitTests/UT_PoolItem.cs @@ -0,0 +1,178 @@ +using System; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Ledger; +using FluentAssertions; +using Neo.Network.P2P.Payloads; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_PoolItem + { + //private PoolItem uut; + private static readonly Random TestRandom = new Random(1337); // use fixed seed for guaranteed determinism + + [TestInitialize] + public void TestSetup() + { + var timeValues = new[] { + new DateTime(1968, 06, 01, 0, 0, 1, DateTimeKind.Utc), + }; + + var timeMock = new Mock(); + timeMock.SetupGet(tp => tp.UtcNow).Returns(() => timeValues[0]) + .Callback(() => timeValues[0] = timeValues[0].Add(TimeSpan.FromSeconds(1))); + TimeProvider.Current = timeMock.Object; + } + + [TestCleanup] + public void TestCleanup() + { + // important to leave TimeProvider correct + TimeProvider.ResetToDefault(); + } + + [TestMethod] + public void PoolItem_CompareTo_ClaimTx() + { + var tx1 = GenerateClaimTx(); + // Non-free low-priority transaction + var tx2 = MockGenerateInvocationTx(new Fixed8(99999), 50).Object; + + var poolItem1 = new PoolItem(tx1); + var poolItem2 = new PoolItem(tx2); + poolItem1.CompareTo(poolItem2).Should().Be(1); + poolItem2.CompareTo(poolItem1).Should().Be(-1); + } + + [TestMethod] + public void PoolItem_CompareTo_Fee() + { + int size1 = 50; + int netFeeSatoshi1 = 1; + var tx1 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshi1), size1); + int size2 = 50; + int netFeeSatoshi2 = 2; + var tx2 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshi2), size2); + + PoolItem pitem1 = new PoolItem(tx1.Object); + PoolItem pitem2 = new PoolItem(tx2.Object); + + Console.WriteLine($"item1 time {pitem1.Timestamp} item2 time {pitem2.Timestamp}"); + // pitem1 < pitem2 (fee) => -1 + pitem1.CompareTo(pitem2).Should().Be(-1); + // pitem2 > pitem1 (fee) => 1 + pitem2.CompareTo(pitem1).Should().Be(1); + } + + [TestMethod] + public void PoolItem_CompareTo_Hash() + { + int sizeFixed = 50; + int netFeeSatoshiFixed = 1; + + for (int testRuns = 0; testRuns < 30; testRuns++) + { + var tx1 = GenerateMockTxWithFirstByteOfHashGreaterThanOrEqualTo(0x80, new Fixed8(netFeeSatoshiFixed), sizeFixed); + var tx2 = GenerateMockTxWithFirstByteOfHashLessThanOrEqualTo(0x79,new Fixed8(netFeeSatoshiFixed), sizeFixed); + + PoolItem pitem1 = new PoolItem(tx1.Object); + PoolItem pitem2 = new PoolItem(tx2.Object); + + // pitem2 < pitem1 (fee) => -1 + pitem2.CompareTo(pitem1).Should().Be(-1); + + // pitem1 > pitem2 (fee) => 1 + pitem1.CompareTo(pitem2).Should().Be(1); + } + + // equal hashes should be equal + var tx3 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + var tx4 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + PoolItem pitem3 = new PoolItem(tx3.Object); + PoolItem pitem4 = new PoolItem(tx4.Object); + + pitem3.CompareTo(pitem4).Should().Be(0); + pitem4.CompareTo(pitem3).Should().Be(0); + } + + [TestMethod] + public void PoolItem_CompareTo_Equals() + { + int sizeFixed = 500; + int netFeeSatoshiFixed = 10; + var tx1 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + var tx2 = MockGenerateInvocationTx(new Fixed8(netFeeSatoshiFixed), sizeFixed, new byte[] {0x13, 0x37}); + + PoolItem pitem1 = new PoolItem(tx1.Object); + PoolItem pitem2 = new PoolItem(tx2.Object); + + // pitem1 == pitem2 (fee) => 0 + pitem1.CompareTo(pitem2).Should().Be(0); + pitem2.CompareTo(pitem1).Should().Be(0); + } + + public Mock GenerateMockTxWithFirstByteOfHashGreaterThanOrEqualTo(byte firstHashByte, Fixed8 networkFee, int size) + { + Mock mockTx; + do + { + mockTx = MockGenerateInvocationTx(networkFee, size); + } while (mockTx.Object.Hash >= new UInt256(TestUtils.GetByteArray(32, firstHashByte))); + + return mockTx; + } + + public Mock GenerateMockTxWithFirstByteOfHashLessThanOrEqualTo(byte firstHashByte, Fixed8 networkFee, int size) + { + Mock mockTx; + do + { + mockTx = MockGenerateInvocationTx(networkFee, size); + } while (mockTx.Object.Hash <= new UInt256(TestUtils.GetByteArray(32, firstHashByte))); + + return mockTx; + } + + public static Transaction GenerateClaimTx() + { + var mockTx = new Mock(); + mockTx.CallBase = true; + mockTx.SetupGet(mr => mr.NetworkFee).Returns(Fixed8.Zero); + mockTx.SetupGet(mr => mr.Size).Returns(50); + var tx = mockTx.Object; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + return mockTx.Object; + } + + // Generate Mock InvocationTransaction with different sizes and prices + public static Mock MockGenerateInvocationTx(Fixed8 networkFee, int size, byte[] overrideScriptBytes=null) + { + var mockTx = new Mock(); + mockTx.CallBase = true; + mockTx.SetupGet(mr => mr.NetworkFee).Returns(networkFee); + mockTx.SetupGet(mr => mr.Size).Returns(size); + + var tx = mockTx.Object; + // use random bytes in the script to get a different hash since we cannot mock the Hash + byte[] randomBytes; + if (overrideScriptBytes != null) + randomBytes = overrideScriptBytes; + else + { + randomBytes = new byte[16]; + TestRandom.NextBytes(randomBytes); + } + tx.Script = randomBytes; + tx.Attributes = new TransactionAttribute[0]; + tx.Inputs = new CoinReference[0]; + tx.Outputs = new TransactionOutput[0]; + tx.Witnesses = new Witness[0]; + return mockTx; + } + } +} diff --git a/neo.UnitTests/UT_UIntBenchmarks.cs b/neo.UnitTests/UT_UIntBenchmarks.cs new file mode 100644 index 0000000000..1c388354a5 --- /dev/null +++ b/neo.UnitTests/UT_UIntBenchmarks.cs @@ -0,0 +1,333 @@ +using FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Neo.Cryptography.ECC; +using Neo.IO; +using Neo.Ledger; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Diagnostics; +using System; +//using System.Runtime.CompilerServices.Unsafe; + +namespace Neo.UnitTests +{ + [TestClass] + public class UT_UIntBenchmarks + { + int MAX_TESTS = 1000000; // 1 million + + byte[][] base_32_1; + byte[][] base_32_2; + byte[][] base_20_1; + byte[][] base_20_2; + + private Random random; + + [TestInitialize] + public void TestSetup() + { + int SEED = 123456789; + random = new Random(SEED); + + base_32_1 = new byte[MAX_TESTS][]; + base_32_2 = new byte[MAX_TESTS][]; + base_20_1 = new byte[MAX_TESTS][]; + base_20_2 = new byte[MAX_TESTS][]; + + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i + { + var checksum = 0; + for(var i=0; i= 0; i--) + { + if (x[i] > y[i]) + return 1; + if (x[i] < y[i]) + return -1; + } + return 0; + } + + private unsafe int code2_UInt256CompareTo(byte[] b1, byte[] b2) + { + fixed (byte* px = b1, py = b2) + { + uint* lpx = (uint*)px; + uint* lpy = (uint*)py; + for (int i = 256/32-1; i >= 0; i--) + { + if (lpx[i] > lpy[i]) + return 1; + if (lpx[i] < lpy[i]) + return -1; + } + } + return 0; + } + + private unsafe int code3_UInt256CompareTo(byte[] b1, byte[] b2) + { + fixed (byte* px = b1, py = b2) + { + ulong* lpx = (ulong*)px; + ulong* lpy = (ulong*)py; + for (int i = 256/64-1; i >= 0; i--) + { + if (lpx[i] > lpy[i]) + return 1; + if (lpx[i] < lpy[i]) + return -1; + } + } + return 0; + } + private int code1_UInt160CompareTo(byte[] b1, byte[] b2) + { + byte[] x = b1; + byte[] y = b2; + for (int i = x.Length - 1; i >= 0; i--) + { + if (x[i] > y[i]) + return 1; + if (x[i] < y[i]) + return -1; + } + return 0; + } + + private unsafe int code2_UInt160CompareTo(byte[] b1, byte[] b2) + { + fixed (byte* px = b1, py = b2) + { + uint* lpx = (uint*)px; + uint* lpy = (uint*)py; + for (int i = 160/32-1; i >= 0; i--) + { + if (lpx[i] > lpy[i]) + return 1; + if (lpx[i] < lpy[i]) + return -1; + } + } + return 0; + } + + private unsafe int code3_UInt160CompareTo(byte[] b1, byte[] b2) + { + // LSB -----------------> MSB + // -------------------------- + // | 8B | 8B | 4B | + // -------------------------- + // 0l 1l 4i + // -------------------------- + fixed (byte* px = b1, py = b2) + { + uint* ipx = (uint*)px; + uint* ipy = (uint*)py; + if (ipx[4] > ipy[4]) + return 1; + if (ipx[4] < ipy[4]) + return -1; + + ulong* lpx = (ulong*)px; + ulong* lpy = (ulong*)py; + if (lpx[1] > lpy[1]) + return 1; + if (lpx[1] < lpy[1]) + return -1; + if (lpx[0] > lpy[0]) + return 1; + if (lpx[0] < lpy[0]) + return -1; + } + return 0; + } + + } +} diff --git a/neo.UnitTests/neo.UnitTests.csproj b/neo.UnitTests/neo.UnitTests.csproj index 3d75fd8e28..158bfba690 100644 --- a/neo.UnitTests/neo.UnitTests.csproj +++ b/neo.UnitTests/neo.UnitTests.csproj @@ -5,6 +5,7 @@ netcoreapp2.0 Neo.UnitTests Neo.UnitTests + true diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index d7d9b1bcfd..924709fc9e 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -202,7 +202,7 @@ public void Reset() public void Fill() { - IEnumerable mem_pool = Blockchain.Singleton.GetMemoryPool(); + IEnumerable mem_pool = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); foreach (IPolicyPlugin plugin in Plugin.Policies) mem_pool = plugin.FilterForBlock(mem_pool); List transactions = mem_pool.ToList(); diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 77b727e0c8..6cd80b38f9 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -219,19 +219,20 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m if (!Crypto.Default.VerifySignature(hashData, context.Signatures[i], context.Validators[i].EncodePoint(false))) context.Signatures[i] = null; context.Signatures[payload.ValidatorIndex] = message.Signature; - Dictionary mempool = Blockchain.Singleton.GetMemoryPool().ToDictionary(p => p.Hash); + Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); + List unverified = new List(); foreach (UInt256 hash in context.TransactionHashes.Skip(1)) { - if (mempool.TryGetValue(hash, out Transaction tx)) + if (mempoolVerified.TryGetValue(hash, out Transaction tx)) { if (!AddTransaction(tx, false)) return; } else { - tx = Blockchain.Singleton.GetUnverifiedTransaction(hash); - if (tx != null) + + if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) unverified.Add(tx); } } diff --git a/neo/IO/Json/JArray.cs b/neo/IO/Json/JArray.cs index 0fc758a0bf..02911d505f 100644 --- a/neo/IO/Json/JArray.cs +++ b/neo/IO/Json/JArray.cs @@ -2,6 +2,7 @@ using System.Collections; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text; namespace Neo.IO.Json @@ -52,6 +53,11 @@ public void Add(JObject item) items.Add(item); } + public override string AsString() + { + return string.Join(",", items.Select(p => p?.AsString())); + } + public void Clear() { items.Clear(); diff --git a/neo/IO/Json/JBoolean.cs b/neo/IO/Json/JBoolean.cs index bf941fc860..6cc39ed231 100644 --- a/neo/IO/Json/JBoolean.cs +++ b/neo/IO/Json/JBoolean.cs @@ -17,18 +17,14 @@ public override bool AsBoolean() return Value; } - public override string AsString() + public override double AsNumber() { - return Value.ToString().ToLower(); + return Value ? 1 : 0; } - public override bool CanConvertTo(Type type) + public override string AsString() { - if (type == typeof(bool)) - return true; - if (type == typeof(string)) - return true; - return false; + return Value.ToString().ToLower(); } internal static JBoolean Parse(TextReader reader) @@ -61,7 +57,7 @@ internal static JBoolean Parse(TextReader reader) public override string ToString() { - return Value.ToString().ToLower(); + return AsString(); } } } diff --git a/neo/IO/Json/JNumber.cs b/neo/IO/Json/JNumber.cs index bf9451975a..fdeb495211 100644 --- a/neo/IO/Json/JNumber.cs +++ b/neo/IO/Json/JNumber.cs @@ -1,6 +1,5 @@ using System; using System.IO; -using System.Reflection; using System.Text; namespace Neo.IO.Json @@ -16,34 +15,7 @@ public JNumber(double value = 0) public override bool AsBoolean() { - if (Value == 0) - return false; - return true; - } - - public override T AsEnum(bool ignoreCase = false) - { - Type t = typeof(T); - TypeInfo ti = t.GetTypeInfo(); - if (!ti.IsEnum) - throw new InvalidCastException(); - if (ti.GetEnumUnderlyingType() == typeof(byte)) - return (T)Enum.ToObject(t, (byte)Value); - if (ti.GetEnumUnderlyingType() == typeof(int)) - return (T)Enum.ToObject(t, (int)Value); - if (ti.GetEnumUnderlyingType() == typeof(long)) - return (T)Enum.ToObject(t, (long)Value); - if (ti.GetEnumUnderlyingType() == typeof(sbyte)) - return (T)Enum.ToObject(t, (sbyte)Value); - if (ti.GetEnumUnderlyingType() == typeof(short)) - return (T)Enum.ToObject(t, (short)Value); - if (ti.GetEnumUnderlyingType() == typeof(uint)) - return (T)Enum.ToObject(t, (uint)Value); - if (ti.GetEnumUnderlyingType() == typeof(ulong)) - return (T)Enum.ToObject(t, (ulong)Value); - if (ti.GetEnumUnderlyingType() == typeof(ushort)) - return (T)Enum.ToObject(t, (ushort)Value); - throw new InvalidCastException(); + return Value != 0 && !double.IsNaN(Value); } public override double AsNumber() @@ -53,23 +25,11 @@ public override double AsNumber() public override string AsString() { + if (double.IsPositiveInfinity(Value)) return "Infinity"; + if (double.IsNegativeInfinity(Value)) return "-Infinity"; return Value.ToString(); } - public override bool CanConvertTo(Type type) - { - if (type == typeof(bool)) - return true; - if (type == typeof(double)) - return true; - if (type == typeof(string)) - return true; - TypeInfo ti = type.GetTypeInfo(); - if (ti.IsEnum && Enum.IsDefined(type, Convert.ChangeType(Value, ti.GetEnumUnderlyingType()))) - return true; - return false; - } - internal static JNumber Parse(TextReader reader) { SkipSpace(reader); @@ -92,7 +52,7 @@ internal static JNumber Parse(TextReader reader) public override string ToString() { - return Value.ToString(); + return AsString(); } public DateTime ToTimestamp() @@ -101,5 +61,21 @@ public DateTime ToTimestamp() throw new InvalidCastException(); return ((ulong)Value).ToDateTime(); } + + public override T TryGetEnum(T defaultValue = default, bool ignoreCase = false) + { + Type enumType = typeof(T); + object value; + try + { + value = Convert.ChangeType(Value, enumType.GetEnumUnderlyingType()); + } + catch (OverflowException) + { + return defaultValue; + } + object result = Enum.ToObject(enumType, value); + return Enum.IsDefined(enumType, result) ? (T)result : defaultValue; + } } } diff --git a/neo/IO/Json/JObject.cs b/neo/IO/Json/JObject.cs index 3d7d9df7b0..a477e49902 100644 --- a/neo/IO/Json/JObject.cs +++ b/neo/IO/Json/JObject.cs @@ -27,55 +27,17 @@ public class JObject public virtual bool AsBoolean() { - throw new InvalidCastException(); - } - - public bool AsBooleanOrDefault(bool value = false) - { - if (!CanConvertTo(typeof(bool))) - return value; - return AsBoolean(); - } - - public virtual T AsEnum(bool ignoreCase = false) - { - throw new InvalidCastException(); - } - - public T AsEnumOrDefault(T value = default(T), bool ignoreCase = false) - { - if (!CanConvertTo(typeof(T))) - return value; - return AsEnum(ignoreCase); + return true; } public virtual double AsNumber() { - throw new InvalidCastException(); - } - - public double AsNumberOrDefault(double value = 0) - { - if (!CanConvertTo(typeof(double))) - return value; - return AsNumber(); + return double.NaN; } public virtual string AsString() { - throw new InvalidCastException(); - } - - public string AsStringOrDefault(string value = null) - { - if (!CanConvertTo(typeof(string))) - return value; - return AsString(); - } - - public virtual bool CanConvertTo(Type type) - { - return false; + return "[object Object]"; } public bool ContainsProperty(string key) @@ -189,6 +151,11 @@ public override string ToString() return sb.ToString(); } + public virtual T TryGetEnum(T defaultValue = default, bool ignoreCase = false) where T : Enum + { + return defaultValue; + } + public static implicit operator JObject(Enum value) { return new JString(value.ToString()); diff --git a/neo/IO/Json/JString.cs b/neo/IO/Json/JString.cs index 3aec381fb4..f208ea95c5 100644 --- a/neo/IO/Json/JString.cs +++ b/neo/IO/Json/JString.cs @@ -1,7 +1,6 @@ using System; using System.Globalization; using System.IO; -using System.Reflection; using System.Text; using System.Text.Encodings.Web; @@ -18,42 +17,13 @@ public JString(string value) public override bool AsBoolean() { - switch (Value.ToLower()) - { - case "0": - case "f": - case "false": - case "n": - case "no": - case "off": - return false; - default: - return true; - } - } - - public override T AsEnum(bool ignoreCase = false) - { - try - { - return (T)Enum.Parse(typeof(T), Value, ignoreCase); - } - catch - { - throw new InvalidCastException(); - } + return !string.IsNullOrEmpty(Value); } public override double AsNumber() { - try - { - return double.Parse(Value); - } - catch - { - throw new InvalidCastException(); - } + if (string.IsNullOrEmpty(Value)) return 0; + return double.TryParse(Value, out double result) ? result : double.NaN; } public override string AsString() @@ -61,19 +31,6 @@ public override string AsString() return Value; } - public override bool CanConvertTo(Type type) - { - if (type == typeof(bool)) - return true; - if (type.GetTypeInfo().IsEnum && Enum.IsDefined(type, Value)) - return true; - if (type == typeof(double)) - return true; - if (type == typeof(string)) - return true; - return false; - } - internal static JString Parse(TextReader reader) { SkipSpace(reader); @@ -89,10 +46,18 @@ internal static JString Parse(TextReader reader) if (c == '\\') { c = (char)reader.Read(); - if (c == 'u') + switch (c) { - reader.Read(buffer, 0, 4); - c = (char)short.Parse(new string(buffer), NumberStyles.HexNumber); + case 'u': + reader.Read(buffer, 0, 4); + c = (char)short.Parse(new string(buffer), NumberStyles.HexNumber); + break; + case 'r': + c = '\r'; + break; + case 'n': + c = '\n'; + break; } } sb.Append(c); @@ -104,5 +69,17 @@ public override string ToString() { return $"\"{JavaScriptEncoder.Default.Encode(Value)}\""; } + + public override T TryGetEnum(T defaultValue = default, bool ignoreCase = false) + { + try + { + return (T)Enum.Parse(typeof(T), Value, ignoreCase); + } + catch + { + return defaultValue; + } + } } } diff --git a/neo/Ledger/AccountState.cs b/neo/Ledger/AccountState.cs index d34738cfb7..b53bcc8e8d 100644 --- a/neo/Ledger/AccountState.cs +++ b/neo/Ledger/AccountState.cs @@ -92,14 +92,14 @@ public override JObject ToJson() JObject json = base.ToJson(); json["script_hash"] = ScriptHash.ToString(); json["frozen"] = IsFrozen; - json["votes"] = new JArray(Votes.Select(p => (JObject)p.ToString())); - json["balances"] = new JArray(Balances.Select(p => + json["votes"] = Votes.Select(p => (JObject)p.ToString()).ToArray(); + json["balances"] = Balances.Select(p => { JObject balance = new JObject(); balance["asset"] = p.Key.ToString(); balance["value"] = p.Value.ToString(); return balance; - })); + }).ToArray(); return json; } } diff --git a/neo/Ledger/Blockchain.cs b/neo/Ledger/Blockchain.cs index 73f123c5af..82dac883a6 100644 --- a/neo/Ledger/Blockchain.cs +++ b/neo/Ledger/Blockchain.cs @@ -1,6 +1,5 @@ using Akka.Actor; using Akka.Configuration; -using Neo.Cryptography; using Neo.Cryptography.ECC; using Neo.IO; using Neo.IO.Actors; @@ -12,10 +11,8 @@ using Neo.SmartContract; using Neo.VM; using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Numerics; using System.Threading; namespace Neo.Ledger @@ -114,19 +111,20 @@ public class ImportCompleted { } } }; + private const int MemoryPoolMaxTransactions = 50_000; + private const int MaxTxToReverifyPerIdle = 10; private static readonly object lockObj = new object(); private readonly NeoSystem system; private readonly List header_index = new List(); private uint stored_header_count = 0; private readonly Dictionary block_cache = new Dictionary(); private readonly Dictionary> block_cache_unverified = new Dictionary>(); - private readonly MemoryPool mem_pool = new MemoryPool(50_000); - private readonly ConcurrentDictionary mem_pool_unverified = new ConcurrentDictionary(); internal readonly RelayCache RelayCache = new RelayCache(100); private readonly HashSet subscribers = new HashSet(); private Snapshot currentSnapshot; public Store Store { get; } + public MemoryPool MemPool { get; } public uint Height => currentSnapshot.Height; public uint HeaderHeight => (uint)header_index.Count - 1; public UInt256 CurrentBlockHash => currentSnapshot.CurrentBlockHash; @@ -150,6 +148,7 @@ static Blockchain() public Blockchain(NeoSystem system, Store store) { this.system = system; + this.MemPool = new MemoryPool(system, MemoryPoolMaxTransactions); this.Store = store; lock (lockObj) { @@ -190,7 +189,7 @@ public bool ContainsBlock(UInt256 hash) public bool ContainsTransaction(UInt256 hash) { - if (mem_pool.ContainsKey(hash)) return true; + if (MemPool.ContainsKey(hash)) return true; return Store.ContainsTransaction(hash); } @@ -218,11 +217,6 @@ public static UInt160 GetConsensusAddress(ECPoint[] validators) return Contract.CreateMultiSigRedeemScript(validators.Length - (validators.Length - 1) / 3, validators).ToScriptHash(); } - public IEnumerable GetMemoryPool() - { - return mem_pool; - } - public Snapshot GetSnapshot() { return Store.GetSnapshot(); @@ -230,17 +224,11 @@ public Snapshot GetSnapshot() public Transaction GetTransaction(UInt256 hash) { - if (mem_pool.TryGetValue(hash, out Transaction transaction)) + if (MemPool.TryGetValue(hash, out Transaction transaction)) return transaction; return Store.GetTransaction(hash); } - internal Transaction GetUnverifiedTransaction(UInt256 hash) - { - mem_pool_unverified.TryGetValue(hash, out Transaction transaction); - return transaction; - } - private void OnImport(IEnumerable blocks) { foreach (Block block in blocks) @@ -264,7 +252,7 @@ private void AddUnverifiedBlockToCache(Block block) blocks.AddLast(block); } - + private RelayResultReason OnNewBlock(Block block) { if (block.Index <= Height) @@ -289,7 +277,7 @@ private RelayResultReason OnNewBlock(Block block) if (block.Index == Height + 1) { Block block_persist = block; - List blocksToPersistList = new List(); + List blocksToPersistList = new List(); while (true) { blocksToPersistList.Add(block_persist); @@ -315,7 +303,7 @@ private RelayResultReason OnNewBlock(Block block) if (block_cache_unverified.TryGetValue(Height + 1, out LinkedList unverifiedBlocks)) { foreach (var unverifiedBlock in unverifiedBlocks) - Self.Tell(unverifiedBlock, ActorRefs.NoSender); + Self.Tell(unverifiedBlock, ActorRefs.NoSender); block_cache_unverified.Remove(Height + 1); } } @@ -385,12 +373,14 @@ private RelayResultReason OnNewTransaction(Transaction transaction) return RelayResultReason.Invalid; if (ContainsTransaction(transaction.Hash)) return RelayResultReason.AlreadyExists; - if (!transaction.Verify(currentSnapshot, GetMemoryPool())) + if (!MemPool.CanTransactionFitInPool(transaction)) + return RelayResultReason.OutOfMemory; + if (!transaction.Verify(currentSnapshot, MemPool.GetVerifiedTransactions())) return RelayResultReason.Invalid; if (!Plugin.CheckPolicy(transaction)) - return RelayResultReason.Unknown; + return RelayResultReason.PolicyFail; - if (!mem_pool.TryAdd(transaction.Hash, transaction)) + if (!MemPool.TryAdd(transaction.Hash, transaction)) return RelayResultReason.OutOfMemory; system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = transaction }); @@ -400,18 +390,7 @@ private RelayResultReason OnNewTransaction(Transaction transaction) private void OnPersistCompleted(Block block) { block_cache.Remove(block.Hash); - foreach (Transaction tx in block.Transactions) - mem_pool.TryRemove(tx.Hash, out _); - mem_pool_unverified.Clear(); - foreach (Transaction tx in mem_pool - .OrderByDescending(p => p.NetworkFee / p.Size) - .ThenByDescending(p => p.NetworkFee) - .ThenByDescending(p => new BigInteger(p.Hash.ToArray()))) - { - mem_pool_unverified.TryAdd(tx.Hash, tx); - Self.Tell(tx, ActorRefs.NoSender); - } - mem_pool.Clear(); + MemPool.UpdatePoolForBlockPersisted(block, currentSnapshot); PersistCompleted completed = new PersistCompleted { Block = block }; system.Consensus?.Tell(completed); Distribute(completed); @@ -439,6 +418,10 @@ protected override void OnReceive(object message) case ConsensusPayload payload: Sender.Tell(OnNewConsensus(payload)); break; + case Idle _: + if (MemPool.ReVerifyTopUnverifiedTransactionsIfNeeded(MaxTxToReverifyPerIdle, currentSnapshot)) + Self.Tell(Idle.Instance, ActorRefs.NoSender); + break; case Terminated terminated: subscribers.Remove(terminated.ActorRef); break; @@ -455,6 +438,7 @@ private void Persist(Block block) { using (Snapshot snapshot = GetSnapshot()) { + List all_application_executed = new List(); snapshot.PersistingBlock = block; snapshot.Blocks.Add(block.Hash, new BlockState { @@ -605,11 +589,15 @@ private void Persist(Block block) break; } if (execution_results.Count > 0) - Distribute(new ApplicationExecuted + { + ApplicationExecuted application_executed = new ApplicationExecuted { Transaction = tx, ExecutionResults = execution_results.ToArray() - }); + }; + Distribute(application_executed); + all_application_executed.Add(application_executed); + } } snapshot.BlockHashIndex.GetAndChange().Hash = block.Hash; snapshot.BlockHashIndex.GetAndChange().Index = block.Index; @@ -620,8 +608,27 @@ private void Persist(Block block) snapshot.HeaderHashIndex.GetAndChange().Index = block.Index; } foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins) - plugin.OnPersist(snapshot); + plugin.OnPersist(snapshot, all_application_executed); snapshot.Commit(); + List commitExceptions = null; + foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins) + { + try + { + plugin.OnCommit(snapshot); + } + catch (Exception ex) + { + if (plugin.ShouldThrowExceptionFromCommit(ex)) + { + if (commitExceptions == null) + commitExceptions = new List(); + + commitExceptions.Add(ex); + } + } + } + if (commitExceptions != null) throw new AggregateException(commitExceptions); } UpdateCurrentSnapshot(); OnPersistCompleted(block); diff --git a/neo/Ledger/MemoryPool.cs b/neo/Ledger/MemoryPool.cs index 28aea62113..edeb66f0f2 100644 --- a/neo/Ledger/MemoryPool.cs +++ b/neo/Ledger/MemoryPool.cs @@ -1,165 +1,511 @@ using Neo.Network.P2P.Payloads; using System; using System.Collections; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Numerics; +using System.Runtime.CompilerServices; +using System.Threading; +using Akka.Util.Internal; +using Neo.Network.P2P; +using Neo.Persistence; +using Neo.Plugins; namespace Neo.Ledger { - internal class MemoryPool : IReadOnlyCollection + public class MemoryPool : IReadOnlyCollection { - private class PoolItem - { - public readonly Transaction Transaction; - public readonly DateTime Timestamp; + // Allow a reverified transaction to be rebroadcasted if it has been this many block times since last broadcast. + private const int BlocksTillRebroadcastLowPriorityPoolTx = 30; + private const int BlocksTillRebroadcastHighPriorityPoolTx = 10; + private int RebroadcastMultiplierThreshold => Capacity / 10; + + private static readonly double MaxSecondsToReverifyHighPrioTx = (double) Blockchain.SecondsPerBlock / 3; + private static readonly double MaxSecondsToReverifyLowPrioTx = (double) Blockchain.SecondsPerBlock / 5; + + // These two are not expected to be hit, they are just safegaurds. + private static readonly double MaxSecondsToReverifyHighPrioTxPerIdle = (double) Blockchain.SecondsPerBlock / 15; + private static readonly double MaxSecondsToReverifyLowPrioTxPerIdle = (double) Blockchain.SecondsPerBlock / 30; + + private readonly NeoSystem _system; + + // + /// + /// Guarantees consistency of the pool data structures. + /// + /// Note: The data structures are only modified from the `Blockchain` actor; so operations guaranteed to be + /// performed by the blockchain actor do not need to acquire the read lock; they only need the write + /// lock for write operations. + /// + private readonly ReaderWriterLockSlim _txRwLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion); + + /// + /// Store all verified unsorted transactions currently in the pool. + /// + private readonly Dictionary _unsortedTransactions = new Dictionary(); + /// + /// Stores the verified high priority sorted transactins currently in the pool. + /// + private readonly SortedSet _sortedHighPrioTransactions = new SortedSet(); + /// + /// Stores the verified low priority sorted transactions currently in the pool. + /// + private readonly SortedSet _sortedLowPrioTransactions = new SortedSet(); + + /// + /// Store the unverified transactions currently in the pool. + /// + /// Transactions in this data structure were valid in some prior block, but may no longer be valid. + /// The top ones that could make it into the next block get verified and moved into the verified data structures + /// (_unsortedTransactions, _sortedLowPrioTransactions, and _sortedHighPrioTransactions) after each block. + /// + private readonly Dictionary _unverifiedTransactions = new Dictionary(); + private readonly SortedSet _unverifiedSortedHighPriorityTransactions = new SortedSet(); + private readonly SortedSet _unverifiedSortedLowPriorityTransactions = new SortedSet(); + + // Internal methods to aid in unit testing + internal int SortedHighPrioTxCount => _sortedHighPrioTransactions.Count; + internal int SortedLowPrioTxCount => _sortedLowPrioTransactions.Count; + internal int UnverifiedSortedHighPrioTxCount => _unverifiedSortedHighPriorityTransactions.Count; + internal int UnverifiedSortedLowPrioTxCount => _unverifiedSortedLowPriorityTransactions.Count; - public PoolItem(Transaction tx) + private int _maxTxPerBlock; + private int _maxLowPriorityTxPerBlock; + + /// + /// Total maximum capacity of transactions the pool can hold. + /// + public int Capacity { get; } + + /// + /// Total count of transactions in the pool. + /// + public int Count + { + get { - Transaction = tx; - Timestamp = DateTime.UtcNow; + _txRwLock.EnterReadLock(); + try + { + return _unsortedTransactions.Count + _unverifiedTransactions.Count; + } + finally + { + _txRwLock.ExitReadLock(); + } } } - private readonly ConcurrentDictionary _mem_pool_fee = new ConcurrentDictionary(); - private readonly ConcurrentDictionary _mem_pool_free = new ConcurrentDictionary(); + /// + /// Total count of verified transactions in the pool. + /// + public int VerifiedCount => _unsortedTransactions.Count; // read of 32 bit type is atomic (no lock) - public int Capacity { get; } - public int Count => _mem_pool_fee.Count + _mem_pool_free.Count; + public int UnVerifiedCount => _unverifiedTransactions.Count; - public MemoryPool(int capacity) + public MemoryPool(NeoSystem system, int capacity) { + _system = system; Capacity = capacity; + LoadMaxTxLimitsFromPolicyPlugins(); + } + + public void LoadMaxTxLimitsFromPolicyPlugins() + { + _maxTxPerBlock = int.MaxValue; + _maxLowPriorityTxPerBlock = int.MaxValue; + foreach (IPolicyPlugin plugin in Plugin.Policies) + { + _maxTxPerBlock = Math.Min(_maxTxPerBlock, plugin.MaxTxPerBlock); + _maxLowPriorityTxPerBlock = Math.Min(_maxLowPriorityTxPerBlock, plugin.MaxLowPriorityTxPerBlock); + } } - public void Clear() + /// + /// Determine whether the pool is holding this transaction and has at some point verified it. + /// Note: The pool may not have verified it since the last block was persisted. To get only the + /// transactions that have been verified during this block use GetVerifiedTransactions() + /// + /// the transaction hash + /// true if the MemoryPool contain the transaction + public bool ContainsKey(UInt256 hash) { - _mem_pool_free.Clear(); - _mem_pool_fee.Clear(); + _txRwLock.EnterReadLock(); + try + { + return _unsortedTransactions.ContainsKey(hash) + || _unverifiedTransactions.ContainsKey(hash); + } + finally + { + _txRwLock.ExitReadLock(); + } } - public bool ContainsKey(UInt256 hash) => _mem_pool_free.ContainsKey(hash) || _mem_pool_fee.ContainsKey(hash); + public bool TryGetValue(UInt256 hash, out Transaction tx) + { + _txRwLock.EnterReadLock(); + try + { + bool ret = _unsortedTransactions.TryGetValue(hash, out PoolItem item) + || _unverifiedTransactions.TryGetValue(hash, out item); + tx = ret ? item.Tx : null; + return ret; + } + finally + { + _txRwLock.ExitReadLock(); + } + } + // Note: This isn't used in Fill during consensus, fill uses GetSortedVerifiedTransactions() public IEnumerator GetEnumerator() { - return - _mem_pool_fee.Select(p => p.Value.Transaction) - .Concat(_mem_pool_free.Select(p => p.Value.Transaction)) - .GetEnumerator(); + _txRwLock.EnterReadLock(); + try + { + return _unsortedTransactions.Select(p => p.Value.Tx) + .Concat(_unverifiedTransactions.Select(p => p.Value.Tx)) + .ToList() + .GetEnumerator(); + } + finally + { + _txRwLock.ExitReadLock(); + } } IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); - static void RemoveLowestFee(ConcurrentDictionary pool, int count) + public IEnumerable GetVerifiedTransactions() { - if (count <= 0) return; - if (count >= pool.Count) + _txRwLock.EnterReadLock(); + try { - pool.Clear(); + return _unsortedTransactions.Select(p => p.Value.Tx).ToArray(); } - else + finally { - UInt256[] delete = pool.AsParallel() - .OrderBy(p => p.Value.Transaction.NetworkFee / p.Value.Transaction.Size) - .ThenBy(p => p.Value.Transaction.NetworkFee) - .ThenBy(p => new BigInteger(p.Key.ToArray())) - .Take(count) - .Select(p => p.Key) - .ToArray(); + _txRwLock.ExitReadLock(); + } + } - foreach (UInt256 hash in delete) - { - pool.TryRemove(hash, out _); - } + public void GetVerifiedAndUnverifiedTransactions(out IEnumerable verifiedTransactions, + out IEnumerable unverifiedTransactions) + { + _txRwLock.EnterReadLock(); + try + { + verifiedTransactions = _sortedHighPrioTransactions.Reverse().Select(p => p.Tx) + .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Tx)).ToArray(); + unverifiedTransactions = _unverifiedSortedHighPriorityTransactions.Reverse().Select(p => p.Tx) + .Concat(_unverifiedSortedLowPriorityTransactions.Reverse().Select(p => p.Tx)).ToArray(); + } + finally + { + _txRwLock.ExitReadLock(); + } + } + + public IEnumerable GetSortedVerifiedTransactions() + { + _txRwLock.EnterReadLock(); + try + { + return _sortedHighPrioTransactions.Reverse().Select(p => p.Tx) + .Concat(_sortedLowPrioTransactions.Reverse().Select(p => p.Tx)) + .ToArray(); + } + finally + { + _txRwLock.ExitReadLock(); } } - static void RemoveOldest(ConcurrentDictionary pool, DateTime time) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private PoolItem GetLowestFeeTransaction(SortedSet verifiedTxSorted, + SortedSet unverifiedTxSorted, out SortedSet sortedPool) + { + PoolItem minItem = unverifiedTxSorted.Min; + sortedPool = minItem != null ? unverifiedTxSorted : null; + + PoolItem verifiedMin = verifiedTxSorted.Min; + if (verifiedMin == null) return minItem; + + if (minItem != null && verifiedMin.CompareTo(minItem) >= 0) + return minItem; + + sortedPool = verifiedTxSorted; + minItem = verifiedMin; + + return minItem; + } + + private PoolItem GetLowestFeeTransaction(out Dictionary unsortedTxPool, out SortedSet sortedPool) { - UInt256[] hashes = pool - .Where(p => p.Value.Timestamp < time) - .Select(p => p.Key) - .ToArray(); + var minItem = GetLowestFeeTransaction(_sortedLowPrioTransactions, _unverifiedSortedLowPriorityTransactions, + out sortedPool); + + if (minItem != null) + { + unsortedTxPool = Object.ReferenceEquals(sortedPool, _unverifiedSortedLowPriorityTransactions) + ? _unverifiedTransactions : _unsortedTransactions; + return minItem; + } - foreach (UInt256 hash in hashes) + try + { + return GetLowestFeeTransaction(_sortedHighPrioTransactions, _unverifiedSortedHighPriorityTransactions, + out sortedPool); + } + finally { - pool.TryRemove(hash, out _); + unsortedTxPool = Object.ReferenceEquals(sortedPool, _unverifiedSortedHighPriorityTransactions) + ? _unverifiedTransactions : _unsortedTransactions; } } - public bool TryAdd(UInt256 hash, Transaction tx) + // Note: this must only be called from a single thread (the Blockchain actor) + internal bool CanTransactionFitInPool(Transaction tx) { - ConcurrentDictionary pool; + if (Count < Capacity) return true; - if (tx.IsLowPriority) + return GetLowestFeeTransaction(out _, out _).CompareTo(tx) <= 0; + } + + /// + /// Adds an already verified transaction to the memory pool. + /// + /// Note: This must only be called from a single thread (the Blockchain actor). To add a transaction to the pool + /// tell the Blockchain actor about the transaction. + /// + /// + /// + /// + internal bool TryAdd(UInt256 hash, Transaction tx) + { + var poolItem = new PoolItem(tx); + + if (_unsortedTransactions.ContainsKey(hash)) return false; + + List removedTransactions = null; + _txRwLock.EnterWriteLock(); + try + { + _unsortedTransactions.Add(hash, poolItem); + + SortedSet pool = tx.IsLowPriority ? _sortedLowPrioTransactions : _sortedHighPrioTransactions; + pool.Add(poolItem); + if (Count > Capacity) + removedTransactions = RemoveOverCapacity(); + } + finally { - pool = _mem_pool_free; + _txRwLock.ExitWriteLock(); } - else + + foreach (IMemoryPoolTxObserverPlugin plugin in Plugin.TxObserverPlugins) { - pool = _mem_pool_fee; + plugin.TransactionAdded(poolItem.Tx); + if (removedTransactions != null) + plugin.TransactionsRemoved(MemoryPoolTxRemovalReason.CapacityExceeded, removedTransactions); } - pool.TryAdd(hash, new PoolItem(tx)); + return _unsortedTransactions.ContainsKey(hash); + } - if (Count > Capacity) + private List RemoveOverCapacity() + { + List removedTransactions = new List(); + do { - RemoveOldest(_mem_pool_free, DateTime.UtcNow.AddSeconds(-Blockchain.SecondsPerBlock * 20)); + PoolItem minItem = GetLowestFeeTransaction(out var unsortedPool, out var sortedPool); + + unsortedPool.Remove(minItem.Tx.Hash); + sortedPool.Remove(minItem); + removedTransactions.Add(minItem.Tx); + } while (Count > Capacity); + + return removedTransactions; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryRemoveVerified(UInt256 hash, out PoolItem item) + { + if (!_unsortedTransactions.TryGetValue(hash, out item)) + return false; + + _unsortedTransactions.Remove(hash); + SortedSet pool = item.Tx.IsLowPriority + ? _sortedLowPrioTransactions : _sortedHighPrioTransactions; + pool.Remove(item); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool TryRemoveUnVerified(UInt256 hash, out PoolItem item) + { + if (!_unverifiedTransactions.TryGetValue(hash, out item)) + return false; + + _unverifiedTransactions.Remove(hash); + SortedSet pool = item.Tx.IsLowPriority + ? _unverifiedSortedLowPriorityTransactions : _unverifiedSortedHighPriorityTransactions; + pool.Remove(item); + return true; + } - var exceed = Count - Capacity; + // Note: this must only be called from a single thread (the Blockchain actor) + internal void UpdatePoolForBlockPersisted(Block block, Snapshot snapshot) + { + _txRwLock.EnterWriteLock(); + try + { + // First remove the transactions verified in the block. + foreach (Transaction tx in block.Transactions) + { + if (TryRemoveVerified(tx.Hash, out _)) continue; + TryRemoveUnVerified(tx.Hash, out _); + } - if (exceed > 0) + // Add all the previously verified transactions back to the unverified transactions + foreach (PoolItem item in _sortedHighPrioTransactions) { - RemoveLowestFee(_mem_pool_free, exceed); - exceed = Count - Capacity; + if (_unverifiedTransactions.TryAdd(item.Tx.Hash, item)) + _unverifiedSortedHighPriorityTransactions.Add(item); + } - if (exceed > 0) - { - RemoveLowestFee(_mem_pool_fee, exceed); - } + foreach (PoolItem item in _sortedLowPrioTransactions) + { + if (_unverifiedTransactions.TryAdd(item.Tx.Hash, item)) + _unverifiedSortedLowPriorityTransactions.Add(item); } + + // Clear the verified transactions now, since they all must be reverified. + _unsortedTransactions.Clear(); + _sortedHighPrioTransactions.Clear(); + _sortedLowPrioTransactions.Clear(); + } + finally + { + _txRwLock.ExitWriteLock(); } - return pool.ContainsKey(hash); + // If we know about headers of future blocks, no point in verifying transactions from the unverified tx pool + // until we get caught up. + if (block.Index > 0 && block.Index < Blockchain.Singleton.HeaderHeight) + return; + + if (Plugin.Policies.Count == 0) + return; + + LoadMaxTxLimitsFromPolicyPlugins(); + + ReverifyTransactions(_sortedHighPrioTransactions, _unverifiedSortedHighPriorityTransactions, + _maxTxPerBlock, MaxSecondsToReverifyHighPrioTx, snapshot); + ReverifyTransactions(_sortedLowPrioTransactions, _unverifiedSortedLowPriorityTransactions, + _maxLowPriorityTxPerBlock, MaxSecondsToReverifyLowPrioTx, snapshot); } - public bool TryRemove(UInt256 hash, out Transaction tx) + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int ReverifyTransactions(SortedSet verifiedSortedTxPool, + SortedSet unverifiedSortedTxPool, int count, double secondsTimeout, Snapshot snapshot) { - if (_mem_pool_free.TryRemove(hash, out PoolItem item)) + DateTime reverifyCutOffTimeStamp = DateTime.UtcNow.AddSeconds(secondsTimeout); + List reverifiedItems = new List(count); + List invalidItems = new List(); + + // Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end. + foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count)) { - tx = item.Transaction; - return true; + if (item.Tx.Verify(snapshot, _unsortedTransactions.Select(p => p.Value.Tx))) + reverifiedItems.Add(item); + else // Transaction no longer valid -- it will be removed from unverifiedTxPool. + invalidItems.Add(item); + + if (DateTime.UtcNow > reverifyCutOffTimeStamp) break; } - else if (_mem_pool_fee.TryRemove(hash, out item)) + + _txRwLock.EnterWriteLock(); + try { - tx = item.Transaction; - return true; + int blocksTillRebroadcast = Object.ReferenceEquals(unverifiedSortedTxPool, _sortedHighPrioTransactions) + ? BlocksTillRebroadcastHighPriorityPoolTx : BlocksTillRebroadcastLowPriorityPoolTx; + + if (Count > RebroadcastMultiplierThreshold) + blocksTillRebroadcast = blocksTillRebroadcast * Count / RebroadcastMultiplierThreshold; + + var rebroadcastCutOffTime = DateTime.UtcNow.AddSeconds( + -Blockchain.SecondsPerBlock * blocksTillRebroadcast); + foreach (PoolItem item in reverifiedItems) + { + if (_unsortedTransactions.TryAdd(item.Tx.Hash, item)) + { + verifiedSortedTxPool.Add(item); + + if (item.LastBroadcastTimestamp < rebroadcastCutOffTime) + { + _system.LocalNode.Tell(new LocalNode.RelayDirectly { Inventory = item.Tx }, _system.Blockchain); + item.LastBroadcastTimestamp = DateTime.UtcNow; + } + } + + _unverifiedTransactions.Remove(item.Tx.Hash); + unverifiedSortedTxPool.Remove(item); + } + + foreach (PoolItem item in invalidItems) + { + _unverifiedTransactions.Remove(item.Tx.Hash); + unverifiedSortedTxPool.Remove(item); + } } - else + finally { - tx = null; - return false; + _txRwLock.ExitWriteLock(); } + + var invalidTransactions = invalidItems.Select(p => p.Tx).ToArray(); + foreach (IMemoryPoolTxObserverPlugin plugin in Plugin.TxObserverPlugins) + plugin.TransactionsRemoved(MemoryPoolTxRemovalReason.NoLongerValid, invalidTransactions); + + return reverifiedItems.Count; } - public bool TryGetValue(UInt256 hash, out Transaction tx) + /// + /// Reverify up to a given maximum count of transactions. Verifies less at a time once the max that can be + /// persisted per block has been reached. + /// + /// Note: this must only be called from a single thread (the Blockchain actor) + /// + /// Max transactions to reverify, the value passed should be >=2. If 1 is passed it + /// will still potentially use 2. + /// The snapshot to use for verifying. + /// true if more unsorted messages exist, otherwise false + internal bool ReVerifyTopUnverifiedTransactionsIfNeeded(int maxToVerify, Snapshot snapshot) { - if (_mem_pool_free.TryGetValue(hash, out PoolItem item)) - { - tx = item.Transaction; - return true; - } - else if (_mem_pool_fee.TryGetValue(hash, out item)) + if (Blockchain.Singleton.Height < Blockchain.Singleton.HeaderHeight) + return false; + + if (_unverifiedSortedHighPriorityTransactions.Count > 0) { - tx = item.Transaction; - return true; + // Always leave at least 1 tx for low priority tx + int verifyCount = _sortedHighPrioTransactions.Count > _maxTxPerBlock || maxToVerify == 1 + ? 1 : maxToVerify - 1; + maxToVerify -= ReverifyTransactions(_sortedHighPrioTransactions, _unverifiedSortedHighPriorityTransactions, + verifyCount, MaxSecondsToReverifyHighPrioTxPerIdle, snapshot); + + if (maxToVerify == 0) maxToVerify++; } - else + + if (_unverifiedSortedLowPriorityTransactions.Count > 0) { - tx = null; - return false; + int verifyCount = _sortedLowPrioTransactions.Count > _maxLowPriorityTxPerBlock + ? 1 : maxToVerify; + ReverifyTransactions(_sortedLowPrioTransactions, _unverifiedSortedLowPriorityTransactions, + verifyCount, MaxSecondsToReverifyLowPrioTxPerIdle, snapshot); } + + return _unverifiedTransactions.Count > 0; } } } diff --git a/neo/Ledger/PoolItem.cs b/neo/Ledger/PoolItem.cs new file mode 100644 index 0000000000..3810330dc4 --- /dev/null +++ b/neo/Ledger/PoolItem.cs @@ -0,0 +1,66 @@ +using Neo.Network.P2P.Payloads; +using System; + +namespace Neo.Ledger +{ + /// + /// Represents an item in the Memory Pool. + /// + // Note: PoolItem objects don't consider transaction priority (low or high) in their compare CompareTo method. + /// This is because items of differing priority are never added to the same sorted set in MemoryPool. + /// + internal class PoolItem : IComparable + { + /// + /// Internal transaction for PoolItem + /// + public readonly Transaction Tx; + + /// + /// Timestamp when transaction was stored on PoolItem + /// + public readonly DateTime Timestamp; + + /// + /// Timestamp when this transaction was last broadcast to other nodes + /// + public DateTime LastBroadcastTimestamp; + + internal PoolItem(Transaction tx) + { + Tx = tx; + Timestamp = TimeProvider.Current.UtcNow; + LastBroadcastTimestamp = Timestamp; + } + + public int CompareTo(Transaction otherTx) + { + if (otherTx == null) return 1; + if (Tx.IsLowPriority && otherTx.IsLowPriority) + { + bool thisIsClaimTx = Tx is ClaimTransaction; + bool otherIsClaimTx = otherTx is ClaimTransaction; + if (thisIsClaimTx != otherIsClaimTx) + { + // This is a claim Tx and other isn't. + if (thisIsClaimTx) return 1; + // The other is claim Tx and this isn't. + return -1; + } + } + // Fees sorted ascending + int ret = Tx.FeePerByte.CompareTo(otherTx.FeePerByte); + if (ret != 0) return ret; + ret = Tx.NetworkFee.CompareTo(otherTx.NetworkFee); + if (ret != 0) return ret; + // Transaction hash sorted descending + return otherTx.Hash.CompareTo(Tx.Hash); + } + + public int CompareTo(PoolItem otherItem) + { + if (otherItem == null) return 1; + return CompareTo(otherItem.Tx); + } + } +} diff --git a/neo/Ledger/RelayResultReason.cs b/neo/Ledger/RelayResultReason.cs index f66504846f..e698d0ea34 100644 --- a/neo/Ledger/RelayResultReason.cs +++ b/neo/Ledger/RelayResultReason.cs @@ -7,6 +7,7 @@ public enum RelayResultReason : byte OutOfMemory, UnableToVerify, Invalid, + PolicyFail, Unknown } } diff --git a/neo/NeoSystem.cs b/neo/NeoSystem.cs index ea0c1a6cdf..efeb8e6f3e 100644 --- a/neo/NeoSystem.cs +++ b/neo/NeoSystem.cs @@ -8,6 +8,7 @@ using Neo.Wallets; using System; using System.Net; +using System.Threading; namespace Neo { @@ -40,10 +41,18 @@ public NeoSystem(Store store) public void Dispose() { RpcServer?.Dispose(); - ActorSystem.Stop(LocalNode); + EnsureStoped(LocalNode); ActorSystem.Dispose(); } + public void EnsureStoped(IActorRef actor) + { + Inbox inbox = Inbox.Create(ActorSystem); + inbox.Watch(actor); + ActorSystem.Stop(actor); + inbox.Receive(Timeout.InfiniteTimeSpan); + } + internal void ResumeNodeStartup() { suspend = false; diff --git a/neo/Network/P2P/ProtocolHandler.cs b/neo/Network/P2P/ProtocolHandler.cs index 572c9b92fe..94766528de 100644 --- a/neo/Network/P2P/ProtocolHandler.cs +++ b/neo/Network/P2P/ProtocolHandler.cs @@ -264,7 +264,7 @@ private void OnInvMessageReceived(InvPayload payload) private void OnMemPoolMessageReceived() { - foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, Blockchain.Singleton.GetMemoryPool().Select(p => p.Hash).ToArray())) + foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => p.Hash).ToArray())) Context.Parent.Tell(Message.Create("inv", payload)); } diff --git a/neo/Network/RPC/RpcServer.cs b/neo/Network/RPC/RpcServer.cs index bd317b10d0..c410690cea 100644 --- a/neo/Network/RPC/RpcServer.cs +++ b/neo/Network/RPC/RpcServer.cs @@ -1,4 +1,14 @@ -using Akka.Actor; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; +using Akka.Actor; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Http; @@ -16,32 +26,24 @@ using Neo.VM; using Neo.Wallets; using Neo.Wallets.NEP6; -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Net; -using System.Net.Security; -using System.Security.Cryptography.X509Certificates; -using System.Text; -using System.Threading.Tasks; namespace Neo.Network.RPC { public sealed class RpcServer : IDisposable { - private readonly NeoSystem system; - private Wallet wallet; + public Wallet Wallet; + private IWebHost host; private Fixed8 maxGasInvoke; + private readonly NeoSystem system; + public static int MAX_CLAIMS_AMOUNT = 50; public static uint DEFAULT_UNLOCK_TIME = 15; - + public RpcServer(NeoSystem system, Wallet wallet = null, Fixed8 maxGasInvoke = default(Fixed8)) { this.system = system; - this.wallet = wallet; + this.Wallet = wallet; this.maxGasInvoke = maxGasInvoke; } @@ -88,7 +90,7 @@ private JObject GetInvokeResult(byte[] script) { json["stack"] = "error: recursive reference"; } - if (wallet != null) + if (Wallet != null) { InvocationTransaction tx = new InvocationTransaction { @@ -99,11 +101,11 @@ private JObject GetInvokeResult(byte[] script) tx.Gas -= Fixed8.FromDecimal(10); if (tx.Gas < Fixed8.Zero) tx.Gas = Fixed8.Zero; tx.Gas = tx.Gas.Ceiling(); - tx = wallet.MakeTransaction(tx); + tx = Wallet.MakeTransaction(tx); if (tx != null) { ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) tx.Witnesses = context.GetWitnesses(); else @@ -128,14 +130,16 @@ private static JObject GetRelayResult(RelayResultReason reason) throw new RpcException(-503, "The block cannot be validated."); case RelayResultReason.Invalid: throw new RpcException(-504, "Block or transaction validation failed."); + case RelayResultReason.PolicyFail: + throw new RpcException(-505, "One of the Policy filters failed."); default: - throw new RpcException(-500, "Unkown error."); + throw new RpcException(-500, "Unknown error."); } } public void OpenWallet(Wallet wallet) { - this.wallet = wallet; + this.Wallet = wallet; } private JObject Process(string method, JArray _params) @@ -198,7 +202,7 @@ private JObject Process(string method, JArray _params) else { UInt160 scriptHash = _params[0].AsString().ToScriptHash(); - WalletAccount account = wallet.GetAccount(scriptHash); + WalletAccount account = Wallet.GetAccount(scriptHash); return account.GetKey().Export(); } case "getaccountstate": @@ -214,7 +218,7 @@ private JObject Process(string method, JArray _params) return asset?.ToJson() ?? throw new RpcException(-100, "Unknown asset"); } case "getbalance": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied."); else { @@ -222,10 +226,10 @@ private JObject Process(string method, JArray _params) switch (UIntBase.Parse(_params[0].AsString())) { case UInt160 asset_id_160: //NEP-5 balance - json["balance"] = wallet.GetAvailable(asset_id_160).ToString(); + json["balance"] = Wallet.GetAvailable(asset_id_160).ToString(); break; case UInt256 asset_id_256: //Global Assets balance - IEnumerable coins = wallet.GetCoins().Where(p => !p.State.HasFlag(CoinState.Spent) && p.Output.AssetId.Equals(asset_id_256)); + IEnumerable coins = Wallet.GetCoins().Where(p => !p.State.HasFlag(CoinState.Spent) && p.Output.AssetId.Equals(asset_id_256)); json["balance"] = coins.Sum(p => p.Output.Value).ToString(); json["confirmed"] = coins.Where(p => p.State.HasFlag(CoinState.Confirmed)).Sum(p => p.Output.Value).ToString(); break; @@ -249,7 +253,7 @@ private JObject Process(string method, JArray _params) } if (block == null) throw new RpcException(-100, "Unknown block"); - bool verbose = _params.Count >= 2 && _params[1].AsBooleanOrDefault(false); + bool verbose = _params.Count >= 2 && _params[1].AsBoolean(); if (verbose) { JObject json = block.ToJson(); @@ -288,7 +292,7 @@ private JObject Process(string method, JArray _params) if (header == null) throw new RpcException(-100, "Unknown block"); - bool verbose = _params.Count >= 2 && _params[1].AsBooleanOrDefault(false); + bool verbose = _params.Count >= 2 && _params[1].AsBoolean(); if (verbose) { JObject json = header.ToJson(); @@ -323,8 +327,8 @@ private JObject Process(string method, JArray _params) throw new RpcException(-400, "Access denied."); else { - WalletAccount account = wallet.CreateAccount(); - if (wallet is NEP6Wallet nep6) + WalletAccount account = Wallet.CreateAccount(); + if (Wallet is NEP6Wallet nep6) nep6.Save(); return account.Address; } @@ -349,11 +353,24 @@ private JObject Process(string method, JArray _params) return json; } case "getrawmempool": - return new JArray(Blockchain.Singleton.GetMemoryPool().Select(p => (JObject)p.Hash.ToString())); + { + bool shouldGetUnverified = _params.Count >= 1 && _params[0].AsBoolean(); + if (!shouldGetUnverified) + return new JArray(Blockchain.Singleton.MemPool.GetVerifiedTransactions().Select(p => (JObject)p.Hash.ToString())); + + JObject json = new JObject(); + json["height"] = Blockchain.Singleton.Height; + Blockchain.Singleton.MemPool.GetVerifiedAndUnverifiedTransactions( + out IEnumerable verifiedTransactions, + out IEnumerable unverifiedTransactions); + json["verified"] = new JArray(verifiedTransactions.Select(p => (JObject) p.Hash.ToString())); + json["unverified"] = new JArray(unverifiedTransactions.Select(p => (JObject) p.Hash.ToString())); + return json; + } case "getrawtransaction": { UInt256 hash = UInt256.Parse(_params[0].AsString()); - bool verbose = _params.Count >= 2 && _params[1].AsBooleanOrDefault(false); + bool verbose = _params.Count >= 2 && _params[1].AsBoolean(); Transaction tx = Blockchain.Singleton.GetTransaction(hash); if (tx == null) throw new RpcException(-100, "Unknown transaction"); @@ -383,6 +400,13 @@ private JObject Process(string method, JArray _params) }) ?? new StorageItem(); return item.Value?.ToHexString(); } + case "gettransactionheight": + { + UInt256 hash = UInt256.Parse(_params[0].AsString()); + uint? height = Blockchain.Singleton.Store.GetTransactions().TryGet(hash)?.BlockIndex; + if (height.HasValue) return height.Value; + throw new RpcException(-100, "Unknown transaction"); + } case "gettxout": { UInt256 hash = UInt256.Parse(_params[0].AsString()); @@ -411,10 +435,10 @@ private JObject Process(string method, JArray _params) return json; } case "getwalletheight": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied."); else - return (wallet.WalletHeight > 0) ? wallet.WalletHeight - 1 : 0; + return (Wallet.WalletHeight > 0) ? Wallet.WalletHeight - 1 : 0; case "invoke": { UInt160 script_hash = UInt160.Parse(_params[0].AsString()); @@ -444,10 +468,10 @@ private JObject Process(string method, JArray _params) return GetInvokeResult(script); } case "listaddress": - if (wallet == null) + if (Wallet == null) throw new RpcException(-400, "Access denied."); else - return wallet.GetAccounts().Select(p => + return Wallet.GetAccounts().Select(p => { JObject account = new JObject(); account["address"] = p.Address; @@ -480,7 +504,7 @@ private JObject Process(string method, JArray _params) if (fee < Fixed8.Zero) throw new RpcException(-32602, "Invalid params"); UInt160 change_address = _params.Count >= 6 ? _params[5].AsString().ToScriptHash() : null; - Transaction tx = wallet.MakeTransaction(null, new[] + Transaction tx = Wallet.MakeTransaction(null, new[] { new TransferOutput { @@ -492,11 +516,11 @@ private JObject Process(string method, JArray _params) if (tx == null) throw new RpcException(-300, "Insufficient funds"); ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) { tx.Witnesses = context.GetWitnesses(); - wallet.ApplyTransaction(tx); + Wallet.ApplyTransaction(tx); system.LocalNode.Tell(new LocalNode.Relay { Inventory = tx }); return tx.ToJson(); } @@ -531,15 +555,15 @@ private JObject Process(string method, JArray _params) if (fee < Fixed8.Zero) throw new RpcException(-32602, "Invalid params"); UInt160 change_address = _params.Count >= 3 ? _params[2].AsString().ToScriptHash() : null; - Transaction tx = wallet.MakeTransaction(null, outputs, change_address: change_address, fee: fee); + Transaction tx = Wallet.MakeTransaction(null, outputs, change_address: change_address, fee: fee); if (tx == null) throw new RpcException(-300, "Insufficient funds"); ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) { tx.Witnesses = context.GetWitnesses(); - wallet.ApplyTransaction(tx); + Wallet.ApplyTransaction(tx); system.LocalNode.Tell(new LocalNode.Relay { Inventory = tx }); return tx.ToJson(); } @@ -569,7 +593,7 @@ private JObject Process(string method, JArray _params) if (fee < Fixed8.Zero) throw new RpcException(-32602, "Invalid params"); UInt160 change_address = _params.Count >= 5 ? _params[4].AsString().ToScriptHash() : null; - Transaction tx = wallet.MakeTransaction(null, new[] + Transaction tx = Wallet.MakeTransaction(null, new[] { new TransferOutput { @@ -581,11 +605,11 @@ private JObject Process(string method, JArray _params) if (tx == null) throw new RpcException(-300, "Insufficient funds"); ContractParametersContext context = new ContractParametersContext(tx); - wallet.Sign(context); + Wallet.Sign(context); if (context.Completed) { tx.Witnesses = context.GetWitnesses(); - wallet.ApplyTransaction(tx); + Wallet.ApplyTransaction(tx); system.LocalNode.Tell(new LocalNode.Relay { Inventory = tx }); return tx.ToJson(); } @@ -745,6 +769,14 @@ private JObject ProcessRequest(HttpContext context, JObject request) if (result == null) result = Process(method, _params); } + catch (FormatException) + { + return CreateErrorResponse(request["id"], -32602, "Invalid params"); + } + catch (IndexOutOfRangeException) + { + return CreateErrorResponse(request["id"], -32602, "Invalid params"); + } catch (Exception ex) { #if DEBUG diff --git a/neo/Plugins/IMemoryPoolTxObserverPlugin.cs b/neo/Plugins/IMemoryPoolTxObserverPlugin.cs new file mode 100644 index 0000000000..e596b9a7b5 --- /dev/null +++ b/neo/Plugins/IMemoryPoolTxObserverPlugin.cs @@ -0,0 +1,11 @@ +using System.Collections.Generic; +using Neo.Network.P2P.Payloads; + +namespace Neo.Plugins +{ + public interface IMemoryPoolTxObserverPlugin + { + void TransactionAdded(Transaction tx); + void TransactionsRemoved(MemoryPoolTxRemovalReason reason, IEnumerable transactions); + } +} diff --git a/neo/Plugins/IPersistencePlugin.cs b/neo/Plugins/IPersistencePlugin.cs index d70afc53fa..af3cbe05b8 100644 --- a/neo/Plugins/IPersistencePlugin.cs +++ b/neo/Plugins/IPersistencePlugin.cs @@ -1,9 +1,14 @@ -using Neo.Persistence; +using System; +using Neo.Persistence; +using System.Collections.Generic; +using static Neo.Ledger.Blockchain; namespace Neo.Plugins { public interface IPersistencePlugin { - void OnPersist(Snapshot snapshot); + void OnPersist(Snapshot snapshot, IReadOnlyList applicationExecutedList); + void OnCommit(Snapshot snapshot); + bool ShouldThrowExceptionFromCommit(Exception ex); } } diff --git a/neo/Plugins/IPolicyPlugin.cs b/neo/Plugins/IPolicyPlugin.cs index 812d418ee2..9952cf2532 100644 --- a/neo/Plugins/IPolicyPlugin.cs +++ b/neo/Plugins/IPolicyPlugin.cs @@ -7,5 +7,7 @@ public interface IPolicyPlugin { bool FilterForMemoryPool(Transaction tx); IEnumerable FilterForBlock(IEnumerable transactions); + int MaxTxPerBlock { get; } + int MaxLowPriorityTxPerBlock { get; } } } diff --git a/neo/Plugins/MemoryPoolTxRemovalReason.cs b/neo/Plugins/MemoryPoolTxRemovalReason.cs new file mode 100644 index 0000000000..f7320dc60e --- /dev/null +++ b/neo/Plugins/MemoryPoolTxRemovalReason.cs @@ -0,0 +1,14 @@ +namespace Neo.Plugins +{ + public enum MemoryPoolTxRemovalReason : byte + { + /// + /// The transaction was ejected since it was the lowest priority transaction and the MemoryPool capacity was exceeded. + /// + CapacityExceeded, + /// + /// The transaction was ejected due to failing re-validation after a block was persisted. + /// + NoLongerValid, + } +} \ No newline at end of file diff --git a/neo/Plugins/Plugin.cs b/neo/Plugins/Plugin.cs index 58a9bb006d..3f32fcb14c 100644 --- a/neo/Plugins/Plugin.cs +++ b/neo/Plugins/Plugin.cs @@ -16,6 +16,7 @@ public abstract class Plugin internal static readonly List Policies = new List(); internal static readonly List RpcPlugins = new List(); internal static readonly List PersistencePlugins = new List(); + internal static readonly List TxObserverPlugins = new List(); private static readonly string pluginsPath = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), "Plugins"); private static readonly FileSystemWatcher configWatcher; @@ -51,6 +52,7 @@ protected Plugin() if (this is IPolicyPlugin policy) Policies.Add(policy); if (this is IRpcPlugin rpc) RpcPlugins.Add(rpc); if (this is IPersistencePlugin persistence) PersistencePlugins.Add(persistence); + if (this is IMemoryPoolTxObserverPlugin txObserver) TxObserverPlugins.Add(txObserver); Configure(); } diff --git a/neo/SmartContract/ContractParameter.cs b/neo/SmartContract/ContractParameter.cs index 0b0f76703e..5e78005e13 100644 --- a/neo/SmartContract/ContractParameter.cs +++ b/neo/SmartContract/ContractParameter.cs @@ -59,7 +59,7 @@ public static ContractParameter FromJson(JObject json) { ContractParameter parameter = new ContractParameter { - Type = json["type"].AsEnum() + Type = json["type"].TryGetEnum() }; if (json["value"] != null) switch (parameter.Type) diff --git a/neo/Wallets/NEP6/NEP6Contract.cs b/neo/Wallets/NEP6/NEP6Contract.cs index d934cf0326..a3dd36f414 100644 --- a/neo/Wallets/NEP6/NEP6Contract.cs +++ b/neo/Wallets/NEP6/NEP6Contract.cs @@ -15,7 +15,7 @@ public static NEP6Contract FromJson(JObject json) return new NEP6Contract { Script = json["script"].AsString().HexToBytes(), - ParameterList = ((JArray)json["parameters"]).Select(p => p["type"].AsEnum()).ToArray(), + ParameterList = ((JArray)json["parameters"]).Select(p => p["type"].TryGetEnum()).ToArray(), ParameterNames = ((JArray)json["parameters"]).Select(p => p["name"].AsString()).ToArray(), Deployed = json["deployed"].AsBoolean() }; diff --git a/neo/Wallets/WalletIndexer.cs b/neo/Wallets/WalletIndexer.cs index 5e1b347ed7..3581dd6728 100644 --- a/neo/Wallets/WalletIndexer.cs +++ b/neo/Wallets/WalletIndexer.cs @@ -127,8 +127,9 @@ public IEnumerable GetTransactions(IEnumerable accounts) yield return hash; } - private void ProcessBlock(Block block, HashSet accounts, WriteBatch batch) + private (Transaction, UInt160[])[] ProcessBlock(Block block, HashSet accounts, WriteBatch batch) { + var change_set = new List<(Transaction, UInt160[])>(); foreach (Transaction tx in block.Transactions) { HashSet accounts_changed = new HashSet(); @@ -218,15 +219,10 @@ private void ProcessBlock(Block block, HashSet accounts, WriteBatch bat { foreach (UInt160 account in accounts_changed) batch.Put(SliceBuilder.Begin(DataEntryPrefix.ST_Transaction).Add(account).Add(tx.Hash), false); - WalletTransaction?.Invoke(null, new WalletTransactionEventArgs - { - Transaction = tx, - RelatedAccounts = accounts_changed.ToArray(), - Height = block.Index, - Time = block.Timestamp - }); + change_set.Add((tx, accounts_changed.ToArray())); } } + return change_set.ToArray(); } private void ProcessBlocks() @@ -234,15 +230,18 @@ private void ProcessBlocks() while (!disposed) { while (!disposed) + { + Block block; + (Transaction, UInt160[])[] change_set; lock (SyncRoot) { if (indexes.Count == 0) break; uint height = indexes.Keys.Min(); - Block block = Blockchain.Singleton.Store.GetBlock(height); + block = Blockchain.Singleton.Store.GetBlock(height); if (block == null) break; WriteBatch batch = new WriteBatch(); HashSet accounts = indexes[height]; - ProcessBlock(block, accounts, batch); + change_set = ProcessBlock(block, accounts, batch); ReadOptions options = ReadOptions.Default; byte[] groupId = db.Get(options, SliceBuilder.Begin(DataEntryPrefix.IX_Group).Add(height)).ToArray(); indexes.Remove(height); @@ -261,6 +260,17 @@ private void ProcessBlocks() } db.Write(WriteOptions.Default, batch); } + foreach (var (tx, accounts) in change_set) + { + WalletTransaction?.Invoke(null, new WalletTransactionEventArgs + { + Transaction = tx, + RelatedAccounts = accounts, + Height = block.Index, + Time = block.Timestamp + }); + } + } for (int i = 0; i < 20 && !disposed; i++) Thread.Sleep(100); } diff --git a/neo/neo.csproj b/neo/neo.csproj index e322842ded..c9752a83fd 100644 --- a/neo/neo.csproj +++ b/neo/neo.csproj @@ -3,7 +3,7 @@ 2015-2018 The Neo Project Neo - 2.9.3 + 2.9.4 The Neo Project netstandard2.0;net47 true @@ -16,6 +16,7 @@ Neo The Neo Project Neo + latest @@ -32,13 +33,13 @@ - - - - - + + + + + - +