Skip to content

Commit

Permalink
Keep track of sender fee (#1183)
Browse files Browse the repository at this point in the history
* Keep track of sender fee to avoid duplicate computation

* Code optimization

* Optimize

* Optimize

* Optimize

* Code optimization

* Correction

* Renaming currentFee to totalSenderFeeFromPool

* Renaming on Verify as well

* Add consideration for null Transactions

* Move sender fee recording systems to class SendersMonitor

* Code optimization

* Capitalize public items

* Code optimization

* Code optimization

* Optimization

* Code optimization
  • Loading branch information
Qiao-Jin authored and vncoelho committed Nov 10, 2019
1 parent f70a76d commit f1d6434
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 14 deletions.
5 changes: 3 additions & 2 deletions neo.UnitTests/Ledger/UT_MemoryPool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;

namespace Neo.UnitTests.Ledger
{
Expand Down Expand Up @@ -73,8 +74,8 @@ private Transaction CreateTransactionWithFee(long fee)
var randomBytes = new byte[16];
random.NextBytes(randomBytes);
Mock<Transaction> mock = new Mock<Transaction>();
mock.Setup(p => p.Reverify(It.IsAny<Snapshot>(), It.IsAny<IEnumerable<Transaction>>())).Returns(true);
mock.Setup(p => p.Verify(It.IsAny<Snapshot>(), It.IsAny<IEnumerable<Transaction>>())).Returns(true);
mock.Setup(p => p.Reverify(It.IsAny<Snapshot>(), It.IsAny<BigInteger>())).Returns(true);
mock.Setup(p => p.Verify(It.IsAny<Snapshot>(), It.IsAny<BigInteger>())).Returns(true);
mock.Object.Script = randomBytes;
mock.Object.Sender = UInt160.Zero;
mock.Object.NetworkFee = fee;
Expand Down
56 changes: 56 additions & 0 deletions neo.UnitTests/Ledger/UT_SendersFeeMonitor.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
using FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Moq;
using Neo.Ledger;
using Neo.Network.P2P.Payloads;
using Neo.Persistence;
using System;
using System.Numerics;

namespace Neo.UnitTests.Ledger
{
[TestClass]
public class UT_SendersFeeMonitor
{
private Transaction CreateTransactionWithFee(long networkFee, long systemFee)
{
Random random = new Random();
var randomBytes = new byte[16];
random.NextBytes(randomBytes);
Mock<Transaction> mock = new Mock<Transaction>();
mock.Setup(p => p.Reverify(It.IsAny<Snapshot>(), It.IsAny<BigInteger>())).Returns(true);
mock.Setup(p => p.Verify(It.IsAny<Snapshot>(), It.IsAny<BigInteger>())).Returns(true);
mock.Object.Script = randomBytes;
mock.Object.Sender = UInt160.Zero;
mock.Object.NetworkFee = networkFee;
mock.Object.SystemFee = systemFee;
mock.Object.Attributes = new TransactionAttribute[0];
mock.Object.Cosigners = new Cosigner[0];
mock.Object.Witnesses = new[]
{
new Witness
{
InvocationScript = new byte[0],
VerificationScript = new byte[0]
}
};
return mock.Object;
}

[TestMethod]
public void TestMemPoolSenderFee()
{
Transaction transaction = CreateTransactionWithFee(1, 2);
SendersFeeMonitor sendersFeeMonitor = new SendersFeeMonitor();
sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(0);
sendersFeeMonitor.AddSenderFee(transaction);
sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(3);
sendersFeeMonitor.AddSenderFee(transaction);
sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(6);
sendersFeeMonitor.RemoveSenderFee(transaction);
sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(3);
sendersFeeMonitor.RemoveSenderFee(transaction);
sendersFeeMonitor.GetSenderFee(transaction.Sender).Should().Be(0);
}
}
}
2 changes: 1 addition & 1 deletion neo.UnitTests/Network/P2P/Payloads/UT_Transaction.cs
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,7 @@ public void Transaction_Reverify_Hashes_Length_Unequal_To_Witnesses_Length()
};
UInt160[] hashes = txSimple.GetScriptHashesForVerifying(snapshot);
Assert.AreEqual(2, hashes.Length);
Assert.IsFalse(txSimple.Reverify(snapshot, new Transaction[0]));
Assert.IsFalse(txSimple.Reverify(snapshot, BigInteger.Zero));
}

[TestMethod]
Expand Down
13 changes: 13 additions & 0 deletions neo/Consensus/ConsensusContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ internal class ConsensusContext : IDisposable, ISerializable
// if this node never heard from validator i, LastSeenMessage[i] will be -1.
public int[] LastSeenMessage;

/// <summary>
/// Store all verified unsorted transactions' senders' fee currently in the consensus context.
/// </summary>
public SendersFeeMonitor SendersFeeMonitor = new SendersFeeMonitor();

public Snapshot Snapshot { get; private set; }
private KeyPair keyPair;
private int _witnessSize;
Expand Down Expand Up @@ -110,6 +115,12 @@ public void Deserialize(BinaryReader reader)
if (TransactionHashes.Length == 0 && !RequestSentOrReceived)
TransactionHashes = null;
Transactions = transactions.Length == 0 && !RequestSentOrReceived ? null : transactions.ToDictionary(p => p.Hash);
SendersFeeMonitor = new SendersFeeMonitor();
if (Transactions != null)
{
foreach (Transaction tx in Transactions.Values)
SendersFeeMonitor.AddSenderFee(tx);
}
}

public void Dispose()
Expand Down Expand Up @@ -245,6 +256,7 @@ internal void EnsureMaxBlockSize(IEnumerable<Transaction> txs)
txs = txs.Take((int)maxTransactionsPerBlock);
List<UInt256> hashes = new List<UInt256>();
Transactions = new Dictionary<UInt256, Transaction>();
SendersFeeMonitor = new SendersFeeMonitor();

// Expected block size
var blockSize = GetExpectedBlockSizeWithoutTransactions(txs.Count());
Expand All @@ -258,6 +270,7 @@ internal void EnsureMaxBlockSize(IEnumerable<Transaction> txs)

hashes.Add(tx.Hash);
Transactions.Add(tx.Hash, tx);
SendersFeeMonitor.AddSenderFee(tx);
}

TransactionHashes = hashes.ToArray();
Expand Down
4 changes: 3 additions & 1 deletion neo/Consensus/ConsensusService.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ internal ConsensusService(IActorRef localNode, IActorRef taskManager, ConsensusC

private bool AddTransaction(Transaction tx, bool verify)
{
if (verify && !tx.Verify(context.Snapshot, context.Transactions.Values))
if (verify && !tx.Verify(context.Snapshot, context.SendersFeeMonitor.GetSenderFee(tx.Sender)))
{
Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning);
RequestChangeView(ChangeViewReason.TxInvalid);
Expand All @@ -74,6 +74,7 @@ private bool AddTransaction(Transaction tx, bool verify)
return false;
}
context.Transactions[tx.Hash] = tx;
context.SendersFeeMonitor.AddSenderFee(tx);
return CheckPrepareResponse();
}

Expand Down Expand Up @@ -423,6 +424,7 @@ private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest m
context.Block.ConsensusData.Nonce = message.Nonce;
context.TransactionHashes = message.TransactionHashes;
context.Transactions = new Dictionary<UInt256, Transaction>();
context.SendersFeeMonitor = new SendersFeeMonitor();
for (int i = 0; i < context.PreparationPayloads.Length; i++)
if (context.PreparationPayloads[i] != null)
if (!context.PreparationPayloads[i].GetDeserializedMessage<PrepareResponse>().PreparationHash.Equals(payload.Hash))
Expand Down
4 changes: 2 additions & 2 deletions neo/Ledger/Blockchain.cs
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ private void OnFillMemoryPool(IEnumerable<Transaction> transactions)
// First remove the tx if it is unverified in the pool.
MemPool.TryRemoveUnVerified(tx.Hash, out _);
// Verify the the transaction
if (!tx.Verify(currentSnapshot, MemPool.GetVerifiedTransactions()))
if (!tx.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(tx.Sender)))
continue;
// Add to the memory pool
MemPool.TryAdd(tx.Hash, tx);
Expand Down Expand Up @@ -370,7 +370,7 @@ private RelayResultReason OnNewTransaction(Transaction transaction, bool relay)
return RelayResultReason.AlreadyExists;
if (!MemPool.CanTransactionFitInPool(transaction))
return RelayResultReason.OutOfMemory;
if (!transaction.Verify(currentSnapshot, MemPool.GetVerifiedTransactions()))
if (!transaction.Verify(currentSnapshot, MemPool.SendersFeeMonitor.GetSenderFee(transaction.Sender)))
return RelayResultReason.Invalid;
if (!NativeContract.Policy.CheckPolicy(transaction, currentSnapshot))
return RelayResultReason.PolicyFail;
Expand Down
11 changes: 10 additions & 1 deletion neo/Ledger/MemoryPool.cs
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ public class MemoryPool : IReadOnlyCollection<Transaction>
/// </summary>
public int Capacity { get; }

/// <summary>
/// Store all verified unsorted transactions' senders' fee currently in the memory pool.
/// </summary>
public SendersFeeMonitor SendersFeeMonitor = new SendersFeeMonitor();

/// <summary>
/// Total count of transactions in the pool.
/// </summary>
Expand Down Expand Up @@ -268,6 +273,7 @@ internal bool TryAdd(UInt256 hash, Transaction tx)
try
{
_unsortedTransactions.Add(hash, poolItem);
SendersFeeMonitor.AddSenderFee(tx);
_sortedTransactions.Add(poolItem);

if (Count > Capacity)
Expand Down Expand Up @@ -310,6 +316,7 @@ private bool TryRemoveVerified(UInt256 hash, out PoolItem item)
return false;

_unsortedTransactions.Remove(hash);
SendersFeeMonitor.RemoveSenderFee(item.Tx);
_sortedTransactions.Remove(item);

return true;
Expand Down Expand Up @@ -337,6 +344,7 @@ internal void InvalidateVerifiedTransactions()

// Clear the verified transactions now, since they all must be reverified.
_unsortedTransactions.Clear();
SendersFeeMonitor = new SendersFeeMonitor();
_sortedTransactions.Clear();
}

Expand Down Expand Up @@ -409,7 +417,7 @@ internal void InvalidateAllTransactions()
// Since unverifiedSortedTxPool is ordered in an ascending manner, we take from the end.
foreach (PoolItem item in unverifiedSortedTxPool.Reverse().Take(count))
{
if (item.Tx.Reverify(snapshot, _unsortedTransactions.Select(p => p.Value.Tx)))
if (item.Tx.Reverify(snapshot, SendersFeeMonitor.GetSenderFee(item.Tx.Sender)))
reverifiedItems.Add(item);
else // Transaction no longer valid -- it will be removed from unverifiedTxPool.
invalidItems.Add(item);
Expand All @@ -432,6 +440,7 @@ internal void InvalidateAllTransactions()
{
if (_unsortedTransactions.TryAdd(item.Tx.Hash, item))
{
SendersFeeMonitor.AddSenderFee(item.Tx);
verifiedSortedTxPool.Add(item);

if (item.LastBroadcastTimestamp < rebroadcastCutOffTime)
Expand Down
43 changes: 43 additions & 0 deletions neo/Ledger/SendersFeeMonitor.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
using Neo.Network.P2P.Payloads;
using System.Collections.Generic;
using System.Numerics;
using System.Threading;

namespace Neo.Ledger
{
public class SendersFeeMonitor
{
private readonly ReaderWriterLockSlim _senderFeeRwLock = new ReaderWriterLockSlim(LockRecursionPolicy.SupportsRecursion);

/// <summary>
/// Store all verified unsorted transactions' senders' fee currently in the memory pool.
/// </summary>
private readonly Dictionary<UInt160, BigInteger> _senderFee = new Dictionary<UInt160, BigInteger>();

public BigInteger GetSenderFee(UInt160 sender)
{
_senderFeeRwLock.EnterReadLock();
if (!_senderFee.TryGetValue(sender, out var value))
value = BigInteger.Zero;
_senderFeeRwLock.ExitReadLock();
return value;
}

public void AddSenderFee(Transaction tx)
{
_senderFeeRwLock.EnterWriteLock();
if (_senderFee.TryGetValue(tx.Sender, out var value))
_senderFee[tx.Sender] = value + tx.SystemFee + tx.NetworkFee;
else
_senderFee.Add(tx.Sender, tx.SystemFee + tx.NetworkFee);
_senderFeeRwLock.ExitWriteLock();
}

public void RemoveSenderFee(Transaction tx)
{
_senderFeeRwLock.EnterWriteLock();
if ((_senderFee[tx.Sender] -= tx.SystemFee + tx.NetworkFee) == 0) _senderFee.Remove(tx.Sender);
_senderFeeRwLock.ExitWriteLock();
}
}
}
12 changes: 5 additions & 7 deletions neo/Network/P2P/Payloads/Transaction.cs
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,14 @@ public UInt160[] GetScriptHashesForVerifying(Snapshot snapshot)
return hashes.OrderBy(p => p).ToArray();
}

public virtual bool Reverify(Snapshot snapshot, IEnumerable<Transaction> mempool)
public virtual bool Reverify(Snapshot snapshot, BigInteger totalSenderFeeFromPool)
{
if (ValidUntilBlock <= snapshot.Height || ValidUntilBlock > snapshot.Height + MaxValidUntilBlockIncrement)
return false;
if (NativeContract.Policy.GetBlockedAccounts(snapshot).Intersect(GetScriptHashesForVerifying(snapshot)).Count() > 0)
return false;
BigInteger balance = NativeContract.GAS.BalanceOf(snapshot, Sender);
BigInteger fee = SystemFee + NetworkFee;
if (balance < fee) return false;
fee += mempool.Where(p => p != this && p.Sender.Equals(Sender)).Select(p => (BigInteger)(p.SystemFee + p.NetworkFee)).Sum();
BigInteger fee = SystemFee + NetworkFee + totalSenderFeeFromPool;
if (balance < fee) return false;
UInt160[] hashes = GetScriptHashesForVerifying(snapshot);
if (hashes.Length != Witnesses.Length) return false;
Expand Down Expand Up @@ -206,12 +204,12 @@ public static Transaction FromJson(JObject json)

bool IInventory.Verify(Snapshot snapshot)
{
return Verify(snapshot, Enumerable.Empty<Transaction>());
return Verify(snapshot, BigInteger.Zero);
}

public virtual bool Verify(Snapshot snapshot, IEnumerable<Transaction> mempool)
public virtual bool Verify(Snapshot snapshot, BigInteger totalSenderFeeFromPool)
{
if (!Reverify(snapshot, mempool)) return false;
if (!Reverify(snapshot, totalSenderFeeFromPool)) return false;
int size = Size;
if (size > MaxTransactionSize) return false;
long net_fee = NetworkFee - size * NativeContract.Policy.GetFeePerByte(snapshot);
Expand Down

0 comments on commit f1d6434

Please sign in to comment.