Perf/eth getlogs with compact encoding

src/Nethermind/Nethermind.Blockchain.Test/Receipts/PersistentReceiptStorageTests.cs

@@ -137,16 +137,8 @@ public void Adds_and_retrieves_receipts_for_block_with_iterator()
             _storage.ClearCache();
             _storage.TryGetReceiptsIterator(block.Number, block.Hash!, out ReceiptsIterator iterator).Should().BeTrue();
             iterator.TryGetNext(out TxReceiptStructRef receiptStructRef).Should().BeTrue();
-            if (_useCompactReceipts)
-            {
-                receiptStructRef.LogsRlp.IsNullOrEmpty().Should().BeTrue();
-                receiptStructRef.Logs.Should().NotBeNullOrEmpty();
-            }
-            else
-            {
-                receiptStructRef.LogsRlp.ToArray().Should().NotBeEmpty();
-                receiptStructRef.Logs.Should().BeNullOrEmpty();
-            }
+            receiptStructRef.LogsRlp.ToArray().Should().NotBeEmpty();
+            receiptStructRef.Logs.Should().BeNullOrEmpty();
 
             iterator.TryGetNext(out receiptStructRef).Should().BeFalse();
         }

src/Nethermind/Nethermind.Blockchain.Test/Receipts/ReceiptsIteratorTests.cs

@@ -0,0 +1,118 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using FluentAssertions;
+using Nethermind.Blockchain.Receipts;
+using Nethermind.Core;
+using Nethermind.Core.Test;
+using Nethermind.Core.Test.Builders;
+using Nethermind.Crypto;
+using Nethermind.Logging;
+using Nethermind.Serialization.Rlp;
+using Nethermind.Specs;
+using NUnit.Framework;
+
+namespace Nethermind.Blockchain.Test.Receipts;
+
+public class ReceiptsIteratorTests
+{
+    ReceiptArrayStorageDecoder _decoder = ReceiptArrayStorageDecoder.Instance;
+
+    [Test]
+    public void SmokeTestWithRecovery()
+    {
+        Block block = Build.A
+            .Block
+            .WithTransactions(3, MainnetSpecProvider.Instance)
+            .TestObject;
+        TxReceipt[] receipts = new[]
+        {
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressA).TestObject,
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressB).TestObject,
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressC).TestObject,
+        };
+
+        ReceiptsIterator iterator = CreateIterator(receipts, block);
+
+        iterator.TryGetNext(out TxReceiptStructRef receipt).Should().BeTrue();
+        iterator.RecoverIfNeeded(ref receipt);
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressA.Bytes);
+        receipt.TxHash.Bytes.ToArray().Should().BeEquivalentTo(block.Transactions[0].Hash.Bytes);
+        iterator.TryGetNext(out receipt).Should().BeTrue();
+        iterator.RecoverIfNeeded(ref receipt);
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressB.Bytes);
+        receipt.TxHash.Bytes.ToArray().Should().BeEquivalentTo(block.Transactions[1].Hash.Bytes);
+        iterator.TryGetNext(out receipt).Should().BeTrue();
+        iterator.RecoverIfNeeded(ref receipt);
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressC.Bytes);
+        receipt.TxHash.Bytes.ToArray().Should().BeEquivalentTo(block.Transactions[1].Hash.Bytes);
+    }
+
+    [Test]
+    public void SmokeTestWithDelayedRecovery()
+    {
+        Block block = Build.A
+            .Block
+            .WithTransactions(3, MainnetSpecProvider.Instance)
+            .TestObject;
+        TxReceipt[] receipts = new[]
+        {
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressA).TestObject,
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressB).TestObject,
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressC).TestObject,
+        };
+
+        ReceiptsIterator iterator = CreateIterator(receipts, block);
+
+        iterator.TryGetNext(out TxReceiptStructRef receipt).Should().BeTrue();
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressA.Bytes);
+        receipt.TxHash.Bytes.Length.Should().Be(0);
+        iterator.TryGetNext(out receipt).Should().BeTrue();
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressB.Bytes);
+        receipt.TxHash.Bytes.Length.Should().Be(0);
+        iterator.TryGetNext(out receipt).Should().BeTrue();
+        iterator.RecoverIfNeeded(ref receipt);
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressC.Bytes);
+        receipt.TxHash.Bytes.ToArray().Should().BeEquivalentTo(block.Transactions[1].Hash.Bytes);
+    }
+
+    [Test]
+    public void SmokeTest()
+    {
+        Block block = Build.A
+            .Block
+            .WithTransactions(3, MainnetSpecProvider.Instance)
+            .TestObject;
+        TxReceipt[] receipts = new[]
+        {
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressA).TestObject,
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressB).TestObject,
+            Build.A.Receipt.WithAllFieldsFilled.WithSender(TestItem.AddressC).TestObject,
+        };
+
+        ReceiptsIterator iterator = CreateIterator(receipts, block);
+
+        iterator.TryGetNext(out TxReceiptStructRef receipt).Should().BeTrue();
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressA.Bytes);
+        iterator.TryGetNext(out receipt).Should().BeTrue();
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressB.Bytes);
+        iterator.TryGetNext(out receipt).Should().BeTrue();
+        receipt.Sender.Bytes.ToArray().Should().BeEquivalentTo(TestItem.AddressC.Bytes);
+    }
+
+    private ReceiptsIterator CreateIterator(TxReceipt[] receipts, Block block)
+    {
+        using NettyRlpStream stream = _decoder.EncodeToNewNettyStream(receipts, RlpBehaviors.Storage);
+        Span<byte> span = stream.AsSpan();
+        TestMemDb blockDb = new TestMemDb();
+        ReceiptsRecovery recovery = new ReceiptsRecovery(
+            new EthereumEcdsa(MainnetSpecProvider.Instance.ChainId, LimboLogs.Instance),
+            MainnetSpecProvider.Instance,
+            false
+        );
+
+        ReceiptsIterator iterator = new ReceiptsIterator(span, blockDb, () => recovery.CreateRecoveryContext(block), _decoder.GetRefDecoder(span));
+        return iterator;
+    }
+}

src/Nethermind/Nethermind.Blockchain/Receipts/KeccaksIterator.cs

@@ -11,20 +11,23 @@ public ref struct KeccaksIterator
     {
         private readonly int _length;
         private Rlp.ValueDecoderContext _decoderContext;
+        private readonly Span<byte> _buffer;
         public long Index { get; private set; }
 
-        public KeccaksIterator(Span<byte> data)
+        public KeccaksIterator(Span<byte> data, Span<byte> buffer)
         {
+            if (buffer.Length != 32) throw new ArgumentException("Buffer must be 32 bytes long");
             _decoderContext = new Rlp.ValueDecoderContext(data);
             _length = _decoderContext.ReadSequenceLength();
+            _buffer = buffer;
             Index = -1;
         }
 
         public bool TryGetNext(out KeccakStructRef current)
         {
             if (_decoderContext.Position < _length)
             {
-                _decoderContext.DecodeKeccakStructRef(out current);
+                _decoderContext.DecodeZeroPrefixedKeccakStructRef(out current, _buffer);
                 Index++;
                 return true;
             }

src/Nethermind/Nethermind.Blockchain/Receipts/LogEntriesIterator.cs

@@ -12,14 +12,16 @@ public ref struct LogEntriesIterator
         private readonly LogEntry[]? _logs;
         private readonly int _length;
         private Rlp.ValueDecoderContext _decoderContext;
+        private IReceiptRefDecoder _receiptRefDecoder;
         public long Index { get; private set; }
 
-        public LogEntriesIterator(Span<byte> data)
+        public LogEntriesIterator(Span<byte> data, IReceiptRefDecoder receiptRefDecoder)
         {
             _decoderContext = new Rlp.ValueDecoderContext(data);
             _length = _decoderContext.ReadSequenceLength();
             Index = -1;
             _logs = null;
+            _receiptRefDecoder = receiptRefDecoder;
         }
 
         public LogEntriesIterator(LogEntry[] logs)
@@ -36,7 +38,7 @@ public bool TryGetNext(out LogEntryStructRef current)
             {
                 if (_decoderContext.Position < _length)
                 {
-                    LogEntryDecoder.DecodeStructRef(ref _decoderContext, RlpBehaviors.None, out current);
+                    _receiptRefDecoder.DecodeLogEntryStructRef(ref _decoderContext, RlpBehaviors.None, out current);
                     Index++;
                     return true;
                 }

src/Nethermind/Nethermind.Blockchain/Receipts/PersistentReceiptStorage.cs

@@ -181,15 +181,22 @@ public bool TryGetReceiptsIterator(long blockNumber, Keccak blockHash, out Recei
 
             var result = CanGetReceiptsByHash(blockNumber);
             var receiptsData = _blocksDb.GetSpan(blockHash);
-            IReceiptsRecovery.IRecoveryContext? recoveryContext = null;
+
+
+            Func<IReceiptsRecovery.IRecoveryContext?> recoveryContextFactory = () => null;
 
             if (_storageDecoder.IsCompactEncoding(receiptsData))
             {
-                Block block = _blockTree.FindBlock(blockHash);
-                recoveryContext = _receiptsRecovery.CreateRecoveryContext(block!);
+                recoveryContextFactory = () =>
+                {
+                    Block block = _blockTree.FindBlock(blockHash);
+                    return _receiptsRecovery.CreateRecoveryContext(block!);
+                };
             }
 
-            iterator = result ? new ReceiptsIterator(receiptsData, _blocksDb, recoveryContext) : new ReceiptsIterator();
+            IReceiptRefDecoder refDecoder = _storageDecoder.GetRefDecoder(receiptsData);
+
+            iterator = result ? new ReceiptsIterator(receiptsData, _blocksDb, recoveryContextFactory, refDecoder) : new ReceiptsIterator();
             return result;
         }
 

src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsIterator.cs

@@ -4,6 +4,7 @@
 using System;
 using Nethermind.Blockchain.Find;
 using Nethermind.Core;
+using Nethermind.Core.Crypto;
 using Nethermind.Db;
 using Nethermind.Serialization.Rlp;
 #pragma warning disable 618
@@ -15,30 +16,33 @@ public ref struct ReceiptsIterator
         private readonly IDbWithSpan _blocksDb;
         private readonly int _length;
         private Rlp.ValueDecoderContext _decoderContext;
+        private int _startingPosition;
 
         private readonly TxReceipt[]? _receipts;
-        private int _position;
-        private readonly IReceiptsRecovery.IRecoveryContext? _recoveryContext;
-        private readonly bool _compactEncoding;
+        private int _receiptIndex;
 
-        public ReceiptsIterator(scoped in Span<byte> receiptsData, IDbWithSpan blocksDb, IReceiptsRecovery.IRecoveryContext? receiptsRecoveryContext)
+        private readonly Func<IReceiptsRecovery.IRecoveryContext>? _recoveryContextFactory;
+        private IReceiptsRecovery.IRecoveryContext? _recoveryContext;
+        private IReceiptRefDecoder _receiptRefDecoder;
+        private bool _recoveryContextConfigured;
+
+        public ReceiptsIterator(scoped in Span<byte> receiptsData, IDbWithSpan blocksDb, Func<IReceiptsRecovery.IRecoveryContext?>? recoveryContextFactory, IReceiptRefDecoder receiptRefDecoder)
         {
             _decoderContext = receiptsData.AsRlpValueContext();
             _blocksDb = blocksDb;
             _receipts = null;
-            _position = 0;
-            _recoveryContext = receiptsRecoveryContext;
+            _receiptIndex = 0;
+            _recoveryContextFactory = recoveryContextFactory;
+            _recoveryContextConfigured = false;
+            _recoveryContext = null;
+            _receiptRefDecoder = receiptRefDecoder;
 
             if (_decoderContext.Length > 0 && _decoderContext.PeekByte() == ReceiptArrayStorageDecoder.CompactEncoding)
             {
-                _compactEncoding = true;
                 _decoderContext.ReadByte();
             }
-            else
-            {
-                _compactEncoding = false;
-            }
 
+            _startingPosition = _decoderContext.Position;
             _length = receiptsData.Length == 0 ? 0 : _decoderContext.ReadSequenceLength();
         }
 
@@ -52,7 +56,8 @@ public ReceiptsIterator(TxReceipt[] receipts)
             _length = receipts.Length;
             _blocksDb = null;
             _receipts = receipts;
-            _position = 0;
+            _receiptIndex = 0;
+            _recoveryContextConfigured = true;
         }
 
         public bool TryGetNext(out TxReceiptStructRef current)
@@ -61,23 +66,17 @@ public bool TryGetNext(out TxReceiptStructRef current)
             {
                 if (_decoderContext.Position < _length)
                 {
-                    if (_compactEncoding)
-                    {
-                        CompactReceiptStorageDecoder.Instance.DecodeStructRef(ref _decoderContext, RlpBehaviors.Storage, out current);
-                    }
-                    else
-                    {
-                        ReceiptStorageDecoder.Instance.DecodeStructRef(ref _decoderContext, RlpBehaviors.Storage, out current);
-                    }
+                    _receiptRefDecoder.DecodeStructRef(ref _decoderContext, RlpBehaviors.Storage, out current);
                     _recoveryContext?.RecoverReceiptData(ref current);
+                    _receiptIndex++;
                     return true;
                 }
             }
             else
             {
-                if (_position < _length)
+                if (_receiptIndex < _length)
                 {
-                    current = new TxReceiptStructRef(_receipts[_position++]);
+                    current = new TxReceiptStructRef(_receipts[_receiptIndex++]);
                     return true;
                 }
             }
@@ -86,17 +85,24 @@ public bool TryGetNext(out TxReceiptStructRef current)
             return false;
         }
 
-        public void Reset()
+        public void RecoverIfNeeded(ref TxReceiptStructRef current)
         {
-            if (_receipts is not null)
-            {
-                _position = 0;
-            }
-            else
+            if (_recoveryContextConfigured) return;
+
+            _recoveryContext = _recoveryContextFactory?.Invoke();
+            if (_recoveryContext != null)
             {
-                _decoderContext.Position = 0;
-                _decoderContext.ReadSequenceLength();
+                // Need to replay the context.
+                _decoderContext.Position = _startingPosition;
+                if (_length != 0) _decoderContext.ReadSequenceLength();
+                for (int i = 0; i < _receiptIndex; i++)
+                {
+                    _receiptRefDecoder.DecodeStructRef(ref _decoderContext, RlpBehaviors.Storage, out current);
+                    _recoveryContext?.RecoverReceiptData(ref current);
+                }
             }
+
+            _recoveryContextConfigured = true;
         }
 
         public void Dispose()
@@ -106,5 +112,17 @@ public void Dispose()
                 _blocksDb?.DangerousReleaseMemory(_decoderContext.Data);
             }
         }
+
+        public LogEntriesIterator IterateLogs(TxReceiptStructRef receipt)
+        {
+            return receipt.Logs is null ? new LogEntriesIterator(receipt.LogsRlp, _receiptRefDecoder) : new LogEntriesIterator(receipt.Logs);
+        }
+
+        public Keccak[] DecodeTopics(Rlp.ValueDecoderContext valueDecoderContext)
+        {
+            return _receiptRefDecoder.DecodeTopics(valueDecoderContext);
+        }
+
+        public bool CanDecodeBloom => _receiptRefDecoder == null || _receiptRefDecoder.CanDecodeBloom;
     }
 }

src/Nethermind/Nethermind.Blockchain/Receipts/ReceiptsRecovery.cs

@@ -55,7 +55,7 @@ public ReceiptsRecoveryResult TryRecover(Block block, TxReceipt[] receipts, bool
             return ReceiptsRecoveryResult.Fail;
         }
 
-        public IReceiptsRecovery.IRecoveryContext CreateRecoveryContext(Block block, bool forceRecoverSender = true)
+        public IReceiptsRecovery.IRecoveryContext CreateRecoveryContext(Block block, bool forceRecoverSender = false)
         {
             var releaseSpec = _specProvider.GetSpec(block.Header);
             return new RecoveryContext(releaseSpec, block, forceRecoverSender, _ecdsa);
@@ -101,7 +101,7 @@ public void RecoverReceiptData(TxReceipt receipt)
                 receipt.BlockNumber = _block.Number;
                 receipt.TxHash = transaction.Hash;
                 receipt.Index = _transactionIndex;
-                receipt.Sender = transaction.SenderAddress ?? (_forceRecoverSender ? _ecdsa.RecoverAddress(transaction, !_releaseSpec.ValidateChainId) : null);
+                receipt.Sender ??= transaction.SenderAddress ?? (_forceRecoverSender ? _ecdsa.RecoverAddress(transaction, !_releaseSpec.ValidateChainId) : null);
                 receipt.Recipient = transaction.IsContractCreation ? null : transaction.To;
 
                 // how would it be in CREATE2?
@@ -129,7 +129,10 @@ public void RecoverReceiptData(ref TxReceiptStructRef receipt)
                 receipt.BlockNumber = _block.Number;
                 receipt.TxHash = transaction.Hash!.ToStructRef();
                 receipt.Index = _transactionIndex;
-                receipt.Sender = (transaction.SenderAddress ?? (_forceRecoverSender ? _ecdsa.RecoverAddress(transaction, !_releaseSpec.ValidateChainId) : Address.Zero))!.ToStructRef();
+                if (receipt.Sender.Bytes == Address.Zero.Bytes)
+                {
+                    receipt.Sender = (transaction.SenderAddress ?? (_forceRecoverSender ? _ecdsa.RecoverAddress(transaction, !_releaseSpec.ValidateChainId) : Address.Zero))!.ToStructRef();
+                }
                 receipt.Recipient = (transaction.IsContractCreation ? Address.Zero : transaction.To)!.ToStructRef();
 
                 // how would it be in CREATE2?

src/Nethermind/Nethermind.Core.Test/Builders/BlockBuilder.cs

@@ -79,6 +79,7 @@ public BlockBuilder WithTransactions(int txCount, ISpecProvider specProvider)
             for (int i = 0; i < txCount; i++)
             {
                 txs[i] = new Transaction();
+                txs[i].Hash = txs[i].CalculateHash();
             }
 
             TxReceipt[] receipts = new TxReceipt[txCount];