ERA history import/export start

src/Nethermind/Directory.Packages.props

@@ -13,6 +13,7 @@
     <PackageVersion Include="Colorful.Console" Version="1.2.15" />
     <PackageVersion Include="CommandLineParser" Version="2.9.1" />
     <PackageVersion Include="ConcurrentHashSet" Version="1.3.0" />
+    <PackageVersion Include="Cortex.SimpleSerialize" Version="0.2.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.0" />
     <PackageVersion Include="coverlet.msbuild" Version="6.0.0" />
     <PackageVersion Include="Crc32.NET" Version="1.2.0" />
@@ -68,6 +69,7 @@
     <PackageVersion Include="Seq.Api" Version="2023.3.0" />
     <PackageVersion Include="SharpCompress" Version="0.34.1" />
     <PackageVersion Include="Shouldly" Version="4.2.1" />
+    <PackageVersion Include="Snappier" Version="1.1.2" />
     <PackageVersion Include="Snappy.Standard" Version="0.2.0" />
     <PackageVersion Include="System.Configuration.ConfigurationManager" Version="7.0.0" />
     <PackageVersion Include="System.Diagnostics.TextWriterTraceListener" Version="4.3.0" />
@@ -83,4 +85,4 @@
     <PackageVersion Include="Websocket.Client" Version="4.6.1" />
     <PackageVersion Include="YamlDotNet" Version="13.1.1" />
   </ItemGroup>
-</Project>
+</Project>

src/Nethermind/Nethermind.Core.Test/Builders/TestItem.cs

@@ -171,5 +171,12 @@ public static byte[] GenerateIndexedAccountRlp(int index, AccountDecoder? accoun
             byte[] value = accountDecoder.Encode(account).Bytes;
             return value;
         }
+
+        public static UInt256 GetRandomAmount(Random? random = null)
+        {
+            Span<byte> buffer = stackalloc byte[32];
+            (random ?? Random).NextBytes(buffer);
+            return new UInt256(buffer);
+        }
     }
 }

src/Nethermind/Nethermind.Core/Collections/ArrayPoolList.cs

@@ -272,4 +272,6 @@ public void Dispose()
     }
 
     public Span<T> AsSpan() => _array.AsSpan(0, _count);
+    public Span<T> AsSpan(int start, int count) => _array.AsSpan(start, count);
+    public Memory<T> AsMemory(int start, int length) => _array.AsMemory(start, length);
 }

src/Nethermind/Nethermind.Era.Test/AccumulatorCalculatorTests.cs

@@ -0,0 +1,46 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Runtime.Intrinsics.Arm;
+using System.Text;
+using System.Threading.Tasks;
+using Nethermind.Core.Crypto;
+
+namespace Nethermind.Era1.Test;
+public class AccumulatorCalculatorTests
+{
+    [Test]
+    public void Add_AddOneHashAndUInt256_DoesNotThrow()
+    {
+        using var sut = new AccumulatorCalculator();
+
+        Assert.That(() => sut.Add(Keccak.Zero, 0), Throws.Nothing);
+    }
+    [Test]
+    public void ComputeRoot_AddValues_ReturnsExpectedResult()
+    {
+        using var sut = new AccumulatorCalculator();
+        sut.Add(Keccak.Zero, 1);
+        sut.Add(Keccak.MaxValue, 2);
+
+        var result = sut.ComputeRoot().ToArray();
+
+        Assert.That(result, Is.EquivalentTo(new[] { 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 }));
+    }
+
+    [Test]
+    public void ComputeRoot_AddOneHashAndUInt256_DoesNotThrow()
+    {
+        using var sut = new AccumulatorCalculator();
+        sut.Add(Keccak.Zero, 1);
+        sut.Add(Keccak.MaxValue, 2);
+
+        var result = sut.ComputeRoot().ToArray();
+
+        Assert.That(result, Is.EquivalentTo(new[] { 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 }));
+    }
+}

src/Nethermind/Nethermind.Era.Test/E2StoreTests.cs

@@ -0,0 +1,258 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using DotNetty.Buffers;
+using FluentAssertions;
+using Nethermind.Serialization.Rlp;
+using Snappier;
+
+namespace Nethermind.Era1.Test;
+internal class E2StoreTests
+{
+    [TestCase(EntryTypes.Version)]
+    [TestCase(EntryTypes.CompressedHeader)]
+    [TestCase(EntryTypes.CompressedBody)]
+    [TestCase(EntryTypes.CompressedReceipts)]
+    [TestCase(EntryTypes.Accumulator)]
+    [TestCase(EntryTypes.BlockIndex)]
+    public async Task WriteEntry_WritingAnEntry_WritesCorrectHeaderType(ushort type)
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+
+        await sut.WriteEntry(type, Array.Empty<byte>());
+
+        Assert.That(BitConverter.ToInt16(stream.ToArray()), Is.EqualTo(type));
+    }
+
+    [TestCase(6)]
+    [TestCase(20)]
+    [TestCase(32)]
+    public async Task WriteEntry_WritingAnEntry_WritesCorrectLengthInHeader(int length)
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+
+        await sut.WriteEntry(EntryTypes.CompressedHeader, new byte[length]);
+
+        Assert.That(BitConverter.ToInt32(stream.ToArray(), 2), Is.EqualTo(length));
+    }
+
+    [TestCase(1)]
+    [TestCase(5)]
+    [TestCase(12)]
+    public async Task WriteEntry_WritingAnEntry_ReturnCorrectNumberofBytesWritten(int length)
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+
+        int result = await sut.WriteEntry(EntryTypes.CompressedHeader, new byte[length]);
+
+        Assert.That(result, Is.EqualTo(length + E2Store.HeaderSize));
+    }
+
+
+    [Test]
+    public async Task WriteEntry_WritingAnEntry_ZeroesAtCorrectIndexesInHeader()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+
+        await sut.WriteEntry(EntryTypes.CompressedHeader, new byte[] { 0xff, 0xff, 0xff, 0xff });
+        byte[] bytes = stream.ToArray();
+
+        Assert.That(bytes[6], Is.EqualTo(0));
+        Assert.That(bytes[7], Is.EqualTo(0));
+    }
+
+    [Test]
+    public async Task WriteEntry_WritingEntryValue_BytesAreWrittenToStream()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+        await sut.WriteEntry(EntryTypes.CompressedHeader, bytes);
+        byte[] result = stream.ToArray();
+
+        Assert.That(new ArraySegment<byte>(result, E2Store.HeaderSize, bytes.Length), Is.EquivalentTo(bytes));
+    }
+
+    [Test]
+    public async Task WriteEntryAsSnappy_WritingEntryValue_WritesEncodedBytesToStream()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+
+        await sut.WriteEntryAsSnappy(EntryTypes.CompressedHeader, bytes);
+        stream.Position = E2Store.HeaderSize;
+        using var snappy = new SnappyStream(stream, System.IO.Compression.CompressionMode.Decompress);
+        byte[] buffer = new byte[32];
+
+        Assert.That(() => snappy.Read(buffer), Throws.Nothing);
+    }
+
+    [Test]
+    public async Task WriteEntryAsSnappy_WritingEntryValue_ReturnsCompressedSize()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+
+        int result = await sut.WriteEntryAsSnappy(EntryTypes.CompressedHeader, bytes);
+
+        Assert.That(result, Is.EqualTo(stream.Length));
+    }
+
+    [Test]
+    public async Task ReadEntryValue_ReadingValueBytesOfEntry_ReturnsBytes()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+        await sut.WriteEntry(EntryTypes.Accumulator, bytes);
+        IByteBuffer buffer = UnpooledByteBufferAllocator.Default.Buffer(bytes.Length);
+        try
+        {
+            int read = await sut.ReadEntryValue(buffer, new Entry(EntryTypes.Accumulator, 0, bytes.Length));
+
+            Assert.That(new ArraySegment<byte>(buffer.Array, buffer.ArrayOffset, read), Is.EquivalentTo(bytes));
+        }
+        finally
+        {
+            buffer.Release();
+        }
+    }
+
+    [Test]
+    public async Task ReadEntryValue_ReadingValueBytesOfEntry_ReturnsBytesRead()
+    {
+        using MemoryStream stream = new MemoryStream();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+        await sut.WriteEntry(EntryTypes.Accumulator, bytes);
+        IByteBuffer buffer = UnpooledByteBufferAllocator.Default.Buffer(bytes.Length);
+
+        int result = await sut.ReadEntryValue(buffer, new Entry(EntryTypes.Accumulator, 0, bytes.Length));
+
+        Assert.That(result, Is.EqualTo(bytes.Length));
+    }
+
+    [Test]
+    public async Task ReadEntryValueAsSnappy_ReadingValueBytesOfEntry_ReturnsDecompressedBytes()
+    {
+        using MemoryStream stream = new();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+        MemoryStream compressed = new();
+        using SnappyStream snappy = new SnappyStream(compressed, System.IO.Compression.CompressionMode.Compress);
+        snappy.Write(bytes);
+        snappy.Flush();
+        byte[] compressedBytes = compressed.ToArray();
+        await sut.WriteEntry(EntryTypes.CompressedHeader, compressedBytes);
+        IByteBuffer buffer = UnpooledByteBufferAllocator.Default.Buffer(32);
+        try
+        {
+            int read = await sut.ReadEntryValueAsSnappy(buffer, new Entry(EntryTypes.CompressedHeader, 0, compressedBytes.Length));
+
+            Assert.That(new ArraySegment<byte>(buffer.Array, 0, read), Is.EquivalentTo(bytes));
+        }
+        finally
+        {
+            buffer.Release();
+        }
+    }
+
+
+    [TestCase(0)]
+    [TestCase(8)]
+    [TestCase(16)]
+    public async Task ReadValueAt_ReadingValueAtDifferentOffset_ReturnsCorrectValue(int offset)
+    {
+        using MemoryStream stream = new();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xf0 };
+        stream.Write(bytes, 0, bytes.Length);
+
+        long result = await sut.ReadValueAt(offset);
+
+        Assert.That(result, Is.EqualTo(BitConverter.ToInt64(bytes, offset)));
+    }
+
+    [TestCase(6)]
+    [TestCase(14)]
+    [TestCase(24)]
+    public async Task ReadEntryAt_ReadingEntryAtDifferentOffset_ReturnsCorrectEntry(int offset)
+    {
+        using MemoryStream stream = new();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0xff, 0xff, 0xff, 0xff };
+        stream.SetLength(offset);
+        stream.Seek(0, SeekOrigin.End);
+        await sut.WriteEntry(EntryTypes.CompressedHeader, bytes);
+
+        Entry result = await sut.ReadEntryAt(offset);
+
+        Assert.That(result.Type, Is.EqualTo(EntryTypes.CompressedHeader));
+        Assert.That(result.Length, Is.EqualTo(bytes.Length));
+        Assert.That(result.Offset, Is.EqualTo(offset));
+    }
+
+    [TestCase(12)]
+    [TestCase(20)]
+    [TestCase(32)]
+    public async Task ReadEntryHeaderAt_ReadingEntryHeaderAtDifferentOffset_ReturnsCorrectEntryHeader(int offset)
+    {
+        using MemoryStream stream = new();
+        using E2Store sut = new E2Store(stream);
+        byte[] bytes = new byte[] { 0xff, 0xff, 0xff, 0xff };
+        stream.SetLength(offset);
+        stream.Seek(0, SeekOrigin.End);
+        await sut.WriteEntry(EntryTypes.CompressedHeader, bytes);
+
+        HeaderData result = await sut.ReadEntryHeaderAt(offset);
+
+        Assert.That(result.Type, Is.EqualTo(EntryTypes.CompressedHeader));
+        Assert.That(result.Length, Is.EqualTo(bytes.Length));
+    }
+
+    [Test]
+    public void Test()
+    {
+        //TODO possible optimization avoid alloc snappy stream by not disposing the stream and reusing like this
+        using MemoryStream stream = new();
+        byte[] bytes = new byte[] { 0x0f, 0xf0, 0xff, 0xff };
+        byte[] bytes1 = new byte[] { 0x1, 0x2, 0x3, 0x4 };
+        MemoryStream compressed = new();
+        using SnappyStream snappy = new SnappyStream(compressed, System.IO.Compression.CompressionMode.Compress);
+        snappy.Write(bytes);
+        snappy.Flush();
+        byte[] compressedBytes = compressed.ToArray();
+        compressed.Seek(0, SeekOrigin.Begin);
+        using SnappyStream snappyDecom = new SnappyStream(compressed, System.IO.Compression.CompressionMode.Decompress);
+        var decomBytes = new byte[bytes.Length];
+        snappyDecom.Read(decomBytes, 0, decomBytes.Length);
+        decomBytes.Should().BeEquivalentTo(bytes);
+
+        compressed.SetLength(0);
+        var snappyHeader = new byte[]
+        {
+            0xff, 0x06, 0x00, 0x00, 0x73, 0x4e, 0x61, 0x50, 0x70, 0x59
+        };
+        compressed.Write(snappyHeader, 0, snappyHeader.Length);
+        snappy.Write(bytes1);
+        snappy.Flush();
+        compressed.Seek(0, SeekOrigin.Begin);
+
+        snappyDecom.Read(decomBytes, 0, decomBytes.Length);
+        decomBytes.Should().BeEquivalentTo(bytes1);
+
+    }
+
+}