Merge pull request #285 from dahlia/states-lazy-evaluation

Allow BlockChain<T> to have incomplete states

CHANGES.md

@@ -10,6 +10,7 @@ To be released.
 
  -  `Peer.AppProtocolVersion` became nullable to represent `Peer` whose version
     is unknown.
+ -  Added `IStore.ListAddresses()` method.  [[#272], [#285]]
 
 ### Added interfaces
 
@@ -18,18 +19,26 @@ To be released.
     `cancellationToken` option.  [[#287]]
  -  Added a `Peer` constructor omitting `appProtocolVersion` parameter
     to create a `Peer` whose version is unknown.
+ -  Added `IncompleteBlockStatesException` class.  [[#272], [#285]]
+ -  Added `completeStates` option to `BlockChain<T>.GetStates()` method.
+    [[#272], [#285]]
 
 ### Behavioral changes
 
  -  `BlockChain<T>.GetNonce()` became to count staged transactions too during
     nonce computation.  [[#270]]
+ -  `BlockChain<T>.GetStates()` method became to throw
+    `IncompleteBlockStatesException` if its `Store` lacks the states of a block
+    that a requested address lastly updated.  [[#272], [#285]]
  -  A message `Swarm` makes became to have multiple blocks within it, which
     means round trips on the network are now much reduced.  [[#273], [#276]]
  -  `Message.Block` has been replaced by `Message.Blocks` and the magic number
     has been changed to `0x0a`.  [[#276]]
  -  Improved performance of `Swarm`'s response time to `GetBlockHashes`
     request messages.  [[#277]]
  -  Added IPv6 support to `Libplanet.Stun.StunAddress`. [[#267], [#271]]
+ -  `IStore.GetBlockStates()` became able to return `null` to represent an absence
+    of states (i.e., incomplete states).  [[#272], [#285]]
 
 ### Bug fixes
 
@@ -48,11 +57,13 @@ To be released.
 [#269]: https://github.com/planetarium/libplanet/pull/269
 [#270]: https://github.com/planetarium/libplanet/pull/270
 [#271]: https://github.com/planetarium/libplanet/pull/271
+[#272]: https://github.com/planetarium/libplanet/issues/272
 [#273]: https://github.com/planetarium/libplanet/issues/273
 [#275]: https://github.com/planetarium/libplanet/pull/275
 [#276]: https://github.com/planetarium/libplanet/pull/276
 [#277]: https://github.com/planetarium/libplanet/pull/277
 [#281]: https://github.com/planetarium/libplanet/pull/281
+[#285]: https://github.com/planetarium/libplanet/pull/285
 [#287]: https://github.com/planetarium/libplanet/pull/287
 
 

Libplanet.Tests/Blockchain/BlockChainTest.cs

@@ -8,6 +8,7 @@
 using Libplanet.Blockchain.Policies;
 using Libplanet.Blocks;
 using Libplanet.Crypto;
+using Libplanet.Store;
 using Libplanet.Tests.Common.Action;
 using Libplanet.Tests.Store;
 using Libplanet.Tx;
@@ -727,6 +728,101 @@ public void GetStatesReturnsEarlyForNonexistentAccount()
             );
         }
 
+        [Fact]
+        public void GetStatesThrowsIncompleteBlockStatesException()
+        {
+            (_, Address[] addresses, BlockChain<DumbAction> chain) =
+                MakeIncompleteBlockStates();
+
+            // As the store has the states for the tip (latest block),
+            // it shouldn't throw an exception.
+            Address lastAddress = addresses.Last();
+            AddressStateMap states = chain.GetStates(new[] { lastAddress });
+            Assert.NotEmpty(states);
+            Assert.Equal("9", states[lastAddress]);
+
+            // As the store lacks the states for blocks other than the tip,
+            // the following GetStates() calls should throw an exception.
+            foreach (Address addr in addresses.Take(addresses.Length - 1))
+            {
+                Assert.Throws<IncompleteBlockStatesException>(() =>
+                    chain.GetStates(new[] { addr })
+                );
+            }
+        }
+
+        [Fact]
+        public void GetStatesWithCompletingStates()
+        {
+            (Address signer, Address[] addresses, BlockChain<DumbAction> chain)
+                = MakeIncompleteBlockStates();
+            string @namespace = chain.Id.ToString();
+            Block<DumbAction>[] blocks = chain.ToArray();
+            StoreTracker store = (StoreTracker)chain.Store;
+
+            HashDigest<SHA256>[] ListStateReferences(Address address)
+            {
+                Block<DumbAction> block = chain.Tip;
+                List<HashDigest<SHA256>> refs = new List<HashDigest<SHA256>>();
+
+                while (true)
+                {
+                    HashDigest<SHA256>? sr =
+                        store.LookupStateReference(@namespace, address, block);
+                    if (sr is HashDigest<SHA256> reference)
+                    {
+                        refs.Add(reference);
+                        block = chain.Blocks[reference];
+                        if (block.PreviousHash is HashDigest<SHA256> prev)
+                        {
+                            block = chain.Blocks[prev];
+                            continue;
+                        }
+                    }
+
+                    break;
+                }
+
+                return refs.ToArray();
+            }
+
+            IImmutableDictionary<Address, HashDigest<SHA256>[]> stateRefs =
+                addresses.Select(a =>
+                    new KeyValuePair<Address, HashDigest<SHA256>[]>(
+                        a,
+                        ListStateReferences(a)
+                    )
+                ).ToImmutableDictionary();
+            long txNonce = store.GetTxNonce(@namespace, signer);
+
+            store.ClearLogs();
+            chain.GetStates(new[] { addresses.Last() }, completeStates: true);
+
+            Assert.Empty(
+                store.Logs.Where(l => l.Item1 == "StoreStateReference")
+            );
+            foreach (Block<DumbAction> block in blocks.Take(blocks.Length - 1))
+            {
+                Assert.Null(store.GetBlockStates(block.Hash));
+            }
+
+            store.ClearLogs();
+            chain.GetStates(new[] { addresses[0] }, completeStates: true);
+
+            foreach (Block<DumbAction> block in blocks)
+            {
+                Assert.NotNull(store.GetBlockStates(block.Hash));
+            }
+
+            // Calculating and filling states should not affect state references
+            // or tx nonce.
+            Assert.Equal(txNonce, store.GetTxNonce(@namespace, signer));
+            foreach (Address address in addresses)
+            {
+                Assert.Equal(stateRefs[address], ListStateReferences(address));
+            }
+        }
+
         [Fact]
         public void EvaluateActions()
         {
@@ -814,6 +910,105 @@ public void GetNonce()
             Assert.Equal(4, _blockChain.GetNonce(address));
         }
 
+        /// <summary>
+        /// Builds a fixture that has incomplete states for blocks other
+        /// than the tip, to test <c>GetStates()</c> method's
+        /// <c>completeStates: true</c> option and
+        /// <see cref="IncompleteBlockStatesException"/>.
+        ///
+        /// <para>The fixture this makes has total 5 addresses (i.e., accounts;
+        /// these go to the second item of the returned triple) and 11 blocks
+        /// (these go to the third item of the returned triple). Every block
+        /// contains a transaction within an action that mutates one account
+        /// state except for the genesis block.  All transactions in the fixture
+        /// are signed by one private key (its address goes to the first item
+        /// of the returned triple).  The most important thing is that
+        /// these blocks all lack its states except for the last block (tip).
+        /// Overall blocks in the fixture look like:</para>
+        ///
+        /// <code>
+        ///  Index   UpdatedAddresses   States in Store
+        /// ------- ------------------ -----------------
+        ///      0                      Absent
+        ///      1   addresses[0]       Absent
+        ///      2   addresses[1]       Absent
+        ///      3   addresses[2]       Absent
+        ///      4   addresses[3]       Absent
+        ///      5   addresses[4]       Absent
+        ///      6   addresses[0]       Absent
+        ///      7   addresses[1]       Absent
+        ///      8   addresses[2]       Absent
+        ///      9   addresses[3]       Absent
+        ///     10   addresses[4]       Present
+        /// </code>
+        /// </summary>
+        private (Address, Address[] addresses, BlockChain<DumbAction> chain)
+            MakeIncompleteBlockStates()
+        {
+            IStore store = new StoreTracker(_fx.Store);
+            Guid chainId = Guid.NewGuid();
+            var chain = new BlockChain<DumbAction>(
+                new NullPolicy<DumbAction>(),
+                store,
+                chainId
+            );
+            var privateKey = new PrivateKey();
+            Address signer = privateKey.PublicKey.ToAddress();
+
+            IImmutableDictionary<Address, object> GetDirty(
+                IEnumerable<ActionEvaluation<DumbAction>> evaluations) =>
+                evaluations.Select(
+                    a => a.OutputStates
+                ).Aggregate(
+                    ImmutableDictionary<Address, object>.Empty,
+                    (x, y) => x.SetItems(y.GetUpdatedStates())
+                );
+
+            // Build the store has incomplete states
+            Block<DumbAction> b = TestUtils.MineGenesis<DumbAction>();
+            chain.Blocks[b.Hash] = b;
+            store.IncreaseTxNonce(chainId.ToString(), b);
+            store.AppendIndex(chainId.ToString(), b.Hash);
+            IImmutableDictionary<Address, object> dirty =
+                GetDirty(b.Evaluate(DateTimeOffset.UtcNow, _ => null));
+            const int accountsCount = 5;
+            Address[] addresses = Enumerable.Repeat<object>(null, accountsCount)
+                .Select(_ => new PrivateKey().PublicKey.ToAddress())
+                .ToArray();
+            for (int i = 0; i < 2; ++i)
+            {
+                for (int j = 0; j < accountsCount; ++j)
+                {
+                    int index = i * accountsCount + j;
+                    Transaction<DumbAction> tx = Transaction<DumbAction>.Create(
+                        store.GetTxNonce(chainId.ToString(), signer),
+                        privateKey,
+                        new[] { new DumbAction(addresses[j], index.ToString()) }
+                    );
+                    b = TestUtils.MineNext(b, new[] { tx });
+                    dirty = GetDirty(
+                        b.Evaluate(
+                            DateTimeOffset.UtcNow,
+                            dirty.GetValueOrDefault
+                        )
+                    );
+                    Assert.NotEmpty(dirty);
+                    chain.Blocks[b.Hash] = b;
+                    store.StoreStateReference(
+                        chainId.ToString(),
+                        dirty.Keys.ToImmutableHashSet(),
+                        b
+                    );
+                    store.IncreaseTxNonce(chainId.ToString(), b);
+                    store.AppendIndex(chainId.ToString(), b.Hash);
+                }
+            }
+
+            store.SetBlockStates(b.Hash, new AddressStateMap(dirty));
+
+            return (signer, addresses, chain);
+        }
+
         private sealed class NullPolicy<T> : IBlockPolicy<T>
             where T : IAction, new()
         {

Libplanet.Tests/Store/StoreTest.cs

@@ -1,5 +1,6 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
+using System.Linq;
 using System.Security.Cryptography;
 using Libplanet.Action;
 using Libplanet.Blocks;
@@ -34,6 +35,43 @@ public void ListNamespaces()
             );
         }
 
+        [Fact]
+        public void ListAddresses()
+        {
+            Assert.Empty(Fx.Store.ListAddresses(Fx.StoreNamespace).ToArray());
+
+            Address[] addresses = Enumerable.Repeat<object>(null, 8)
+                .Select(_ => new PrivateKey().PublicKey.ToAddress())
+                .ToArray();
+            Fx.Store.StoreStateReference(
+                Fx.StoreNamespace,
+                addresses.Take(3).ToImmutableHashSet(),
+                Fx.Block1
+            );
+            Assert.Equal(
+                addresses.Take(3).ToImmutableHashSet(),
+                Fx.Store.ListAddresses(Fx.StoreNamespace).ToImmutableHashSet()
+            );
+            Fx.Store.StoreStateReference(
+                Fx.StoreNamespace,
+                addresses.Skip(2).Take(3).ToImmutableHashSet(),
+                Fx.Block2
+            );
+            Assert.Equal(
+                addresses.Take(5).ToImmutableHashSet(),
+                Fx.Store.ListAddresses(Fx.StoreNamespace).ToImmutableHashSet()
+            );
+            Fx.Store.StoreStateReference(
+                Fx.StoreNamespace,
+                addresses.Skip(5).Take(3).ToImmutableHashSet(),
+                Fx.Block3
+            );
+            Assert.Equal(
+                addresses.ToImmutableHashSet(),
+                Fx.Store.ListAddresses(Fx.StoreNamespace).ToImmutableHashSet()
+            );
+        }
+
         [Fact]
         public void StoreBlock()
         {
@@ -335,9 +373,9 @@ public void StoreStage()
         }
 
         [Fact]
-        public void StoreBlockState()
+        public void BlockState()
         {
-            Assert.Empty(Fx.Store.GetBlockStates(Fx.Hash1));
+            Assert.Null(Fx.Store.GetBlockStates(Fx.Hash1));
             AddressStateMap states = new AddressStateMap(
                 new Dictionary<Address, object>()
                 {

Libplanet.Tests/Store/StoreTracker.cs

@@ -61,6 +61,12 @@ public bool DeleteIndex(string @namespace, HashDigest<SHA256> hash)
             return _store.DeleteIndex(@namespace, hash);
         }
 
+        public IEnumerable<Address> ListAddresses(string @namespace)
+        {
+            _logs.Add((nameof(ListAddresses), @namespace, null));
+            return _store.ListAddresses(@namespace);
+        }
+
         public bool DeleteTransaction(TxId txid)
         {
             _logs.Add((nameof(DeleteTransaction), txid, null));

Libplanet/Address.cs

@@ -253,7 +253,8 @@ private static byte[] DeriveAddress(string hex)
             if (hex.Length != 40)
             {
                 throw new ArgumentException(
-                    "address hex must be 40 bytes",
+                    "Address hex must be 40 bytes, but " +
+                    $"{hex.Length} bytes were passed.",
                     nameof(hex)
                 );
             }