[neox] persistence (neo-project#1721)

* don't use snapshot directly in plugin

* persistence and ut

* fix mpt

* precommit

* fix commit

* rm LocalRootHashIndex

* fix ut

* pre commit

* fix clone view

* change name

* rename

* abstract

* comment

* fix ReadOnlyView

* rm precommit

* rm HashState

* add MPTDataCache

* optimze

* optimize

* remove blank line

* StateRoot verify fee

* expose Root in MPTDataCache

* fix some and ut

* merge master

* master

* fix mpt ut

* fix Storages and name

* rename

* add comment

* proof prefix

* fix

* format

* format

* format

* add StateRoot ut

* reset mpt prefix

* rm GetMessage

* throw exception when no script hash in state root

* UpdateLocalStateRoot when Storages changed

* rename

Co-authored-by: Tommo-L <luchuan@neo.org>

src/neo/Cryptography/MPT/MPTTrie.cs

@@ -18,7 +18,7 @@ public partial class MPTTrie<TKey, TValue>
         public MPTTrie(ISnapshot store, UInt256 root)
         {
             this.store = store ?? throw new ArgumentNullException();
-            this.root = root is null ? HashNode.EmptyNode : new HashNode(root);
+            this.root = root is null || root == UInt256.Zero ? HashNode.EmptyNode : new HashNode(root);
         }
 
         private MPTNode Resolve(HashNode n)

src/neo/Ledger/Blockchain.cs

@@ -445,6 +445,7 @@ private void Persist(Block block)
                 snapshot.BlockHashIndex.GetAndChange().Set(block);
                 foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins)
                     plugin.OnPersist(snapshot, all_application_executed);
+                snapshot.UpdateLocalStateRoot();
                 snapshot.Commit();
                 List<Exception> commitExceptions = null;
                 foreach (IPersistencePlugin plugin in Plugin.PersistencePlugins)

src/neo/Network/P2P/MessageCommand.cs

@@ -40,6 +40,8 @@ public enum MessageCommand : byte
         Block = 0x2c,
         [ReflectionCache(typeof(ConsensusPayload))]
         Consensus = 0x2d,
+        [ReflectionCache(typeof(StateRoot))]
+        StateRoot = 0x2e,
         Reject = 0x2f,
 
         //SPV protocol

src/neo/Network/P2P/Payloads/InventoryType.cs

@@ -4,6 +4,7 @@ public enum InventoryType : byte
     {
         TX = MessageCommand.Transaction,
         Block = MessageCommand.Block,
+        StateRoot = MessageCommand.StateRoot,
         Consensus = MessageCommand.Consensus
     }
 }

src/neo/Network/P2P/Payloads/StateRoot.cs

@@ -0,0 +1,122 @@
+using Neo.Cryptography;
+using Neo.IO;
+using Neo.IO.Json;
+using Neo.Ledger;
+using Neo.Persistence;
+using Neo.SmartContract;
+using Neo.SmartContract.Native;
+using System;
+using System.IO;
+
+namespace Neo.Network.P2P.Payloads
+{
+    public class StateRoot : ICloneable<StateRoot>, IInventory
+    {
+        public byte Version;
+        public uint Index;
+        public UInt256 RootHash;
+        public Witness Witness;
+
+        InventoryType IInventory.InventoryType => InventoryType.StateRoot;
+
+        private UInt256 _hash = null;
+
+        public UInt256 Hash
+        {
+            get
+            {
+                if (_hash == null)
+                {
+                    _hash = new UInt256(Crypto.Hash256(this.GetHashData()));
+                }
+                return _hash;
+            }
+        }
+
+        Witness[] IVerifiable.Witnesses
+        {
+            get
+            {
+                return new[] { Witness };
+            }
+            set
+            {
+                if (value.Length != 1) throw new ArgumentException();
+                Witness = value[0];
+            }
+        }
+
+        public int Size =>
+            sizeof(byte) +          //Version
+            sizeof(uint) +          //Index
+            UInt256.Length +        //Root
+            Witness.Size;           //Witness
+
+        StateRoot ICloneable<StateRoot>.Clone()
+        {
+            return new StateRoot
+            {
+                Version = Version,
+                Index = Index,
+                RootHash = RootHash,
+                Witness = Witness,
+            };
+        }
+
+        void ICloneable<StateRoot>.FromReplica(StateRoot replica)
+        {
+            Version = replica.Version;
+            Index = replica.Index;
+            RootHash = replica.RootHash;
+            Witness = replica.Witness;
+        }
+
+        public void Deserialize(BinaryReader reader)
+        {
+            this.DeserializeUnsigned(reader);
+            Witness = reader.ReadSerializable<Witness>();
+        }
+
+        public void DeserializeUnsigned(BinaryReader reader)
+        {
+            Version = reader.ReadByte();
+            Index = reader.ReadUInt32();
+            RootHash = reader.ReadSerializable<UInt256>();
+        }
+
+        public void Serialize(BinaryWriter writer)
+        {
+            this.SerializeUnsigned(writer);
+            writer.Write(Witness);
+        }
+
+        public void SerializeUnsigned(BinaryWriter writer)
+        {
+            writer.Write(Version);
+            writer.Write(Index);
+            writer.Write(RootHash);
+        }
+
+        public bool Verify(StoreView snapshot)
+        {
+            return this.VerifyWitnesses(snapshot, 1_00000000);
+        }
+
+        public virtual UInt160[] GetScriptHashesForVerifying(StoreView snapshot)
+        {
+            var script_hash = Blockchain.Singleton.GetBlock(Index)?.NextConsensus;
+            if (script_hash is null) throw new System.InvalidOperationException("No script hash for state root verifying");
+            return new UInt160[] { script_hash };
+        }
+
+        public JObject ToJson()
+        {
+            var json = new JObject();
+            json["version"] = Version;
+            json["index"] = Index;
+            json["stateroot"] = RootHash.ToString();
+            json["witness"] = Witness.ToJson();
+            return json;
+        }
+    }
+}

src/neo/Persistence/ClonedView.cs

@@ -1,6 +1,8 @@
+using Neo.Cryptography.MPT;
 using Neo.IO;
 using Neo.IO.Caching;
 using Neo.Ledger;
+using Neo.Network.P2P.Payloads;
 
 namespace Neo.Persistence
 {
@@ -11,8 +13,10 @@ internal class ClonedView : StoreView
         public override DataCache<UInt160, ContractState> Contracts { get; }
         public override DataCache<StorageKey, StorageItem> Storages { get; }
         public override DataCache<SerializableWrapper<uint>, HeaderHashList> HeaderHashList { get; }
+        public override DataCache<SerializableWrapper<uint>, HashIndexState> LocalStateRoot { get; }
         public override MetaDataCache<HashIndexState> BlockHashIndex { get; }
         public override MetaDataCache<HashIndexState> HeaderHashIndex { get; }
+        public override MetaDataCache<StateRoot> ValidatorsStateRoot { get; }
         public override MetaDataCache<ContractIdState> ContractId { get; }
 
         public ClonedView(StoreView view)
@@ -23,8 +27,10 @@ public ClonedView(StoreView view)
             this.Contracts = view.Contracts.CreateSnapshot();
             this.Storages = view.Storages.CreateSnapshot();
             this.HeaderHashList = view.HeaderHashList.CreateSnapshot();
+            this.LocalStateRoot = view.LocalStateRoot.CreateSnapshot();
             this.BlockHashIndex = view.BlockHashIndex.CreateSnapshot();
             this.HeaderHashIndex = view.HeaderHashIndex.CreateSnapshot();
+            this.ValidatorsStateRoot = view.ValidatorsStateRoot.CreateSnapshot();
             this.ContractId = view.ContractId.CreateSnapshot();
         }
     }

src/neo/Persistence/Helper.cs

@@ -1,4 +1,5 @@
 using System;
+using Neo.Ledger;
 
 namespace Neo.Persistence
 {
@@ -8,5 +9,13 @@ public static byte[] EnsureNotNull(this byte[] source)
         {
             return source ?? Array.Empty<byte>();
         }
+
+        public static void UpdateLocalStateRoot(this SnapshotView snapshot)
+        {
+            snapshot.Storages.Commit();
+            var root = snapshot.LocalStateRoot.GetAndChange(snapshot.Height, () => new HashIndexState());
+            root.Index = snapshot.Height;
+            root.Hash = ((MPTDataCache<StorageKey, StorageItem>)snapshot.Storages).Root.Hash;
+        }
     }
 }

src/neo/Persistence/MPTDataCache.cs

@@ -0,0 +1,53 @@
+
+using Neo.Cryptography.MPT;
+using Neo.IO;
+using Neo.IO.Caching;
+using System;
+using System.Collections.Generic;
+
+namespace Neo.Persistence
+{
+    internal class MPTDataCache<TKey, TValue> : DataCache<TKey, TValue>
+    where TKey : IEquatable<TKey>, ISerializable, new()
+    where TValue : class, ICloneable<TValue>, ISerializable, new()
+    {
+        private MPTTrie<TKey, TValue> mptTrie;
+
+        public MPTNode Root => mptTrie.Root;
+
+        public MPTDataCache(IReadOnlyStore store, UInt256 root)
+        {
+            mptTrie = new MPTTrie<TKey, TValue>(store as ISnapshot, root);
+        }
+
+        protected override void AddInternal(TKey key, TValue value)
+        {
+            mptTrie.Put(key, value);
+        }
+
+        protected override void DeleteInternal(TKey key)
+        {
+            mptTrie.Delete(key);
+        }
+
+        protected override IEnumerable<(TKey Key, TValue Value)> FindInternal(byte[] key_prefix)
+        {
+            return mptTrie.Find(key_prefix);
+        }
+
+        protected override TValue GetInternal(TKey key)
+        {
+            return mptTrie[key];
+        }
+
+        protected override TValue TryGetInternal(TKey key)
+        {
+            return mptTrie[key];
+        }
+
+        protected override void UpdateInternal(TKey key, TValue value)
+        {
+            mptTrie.Put(key, value);
+        }
+    }
+}

src/neo/Persistence/Prefixes.cs

@@ -6,12 +6,13 @@ internal static class Prefixes
         public const byte DATA_Transaction = 0x02;
 
         public const byte ST_Contract = 0x50;
-        public const byte ST_Storage = 0x70;
+        public const byte ST_LocalStateRoot = 0x60;
 
         public const byte IX_HeaderHashList = 0x80;
         public const byte IX_CurrentBlock = 0xc0;
         public const byte IX_CurrentHeader = 0xc1;
         public const byte IX_ContractId = 0xc2;
+        public const byte IX_ValidatorsStateRoot = 0xc4;
 
         /* Prefixes 0xf0 to 0xff are reserved for external use.
          *

src/neo/Persistence/ReadOnlyView.cs

@@ -1,6 +1,8 @@
+using Neo.Cryptography.MPT;
 using Neo.IO;
 using Neo.IO.Caching;
 using Neo.Ledger;
+using Neo.Network.P2P.Payloads;
 using System;
 
 namespace Neo.Persistence
@@ -15,10 +17,12 @@ public class ReadOnlyView : StoreView
         public override DataCache<UInt256, TrimmedBlock> Blocks => new StoreDataCache<UInt256, TrimmedBlock>(store, Prefixes.DATA_Block);
         public override DataCache<UInt256, TransactionState> Transactions => new StoreDataCache<UInt256, TransactionState>(store, Prefixes.DATA_Transaction);
         public override DataCache<UInt160, ContractState> Contracts => new StoreDataCache<UInt160, ContractState>(store, Prefixes.ST_Contract);
-        public override DataCache<StorageKey, StorageItem> Storages => new StoreDataCache<StorageKey, StorageItem>(store, Prefixes.ST_Storage);
+        public override DataCache<StorageKey, StorageItem> Storages => new MPTDataCache<StorageKey, StorageItem>(store, CurrentStateRootHash);
         public override DataCache<SerializableWrapper<uint>, HeaderHashList> HeaderHashList => new StoreDataCache<SerializableWrapper<uint>, HeaderHashList>(store, Prefixes.IX_HeaderHashList);
+        public override DataCache<SerializableWrapper<uint>, HashIndexState> LocalStateRoot => new StoreDataCache<SerializableWrapper<uint>, HashIndexState>(store, Prefixes.ST_LocalStateRoot);
         public override MetaDataCache<HashIndexState> BlockHashIndex => new StoreMetaDataCache<HashIndexState>(store, Prefixes.IX_CurrentBlock);
         public override MetaDataCache<HashIndexState> HeaderHashIndex => new StoreMetaDataCache<HashIndexState>(store, Prefixes.IX_CurrentHeader);
+        public override MetaDataCache<StateRoot> ValidatorsStateRoot => new StoreMetaDataCache<StateRoot>(store, Prefixes.IX_ValidatorsStateRoot);
         public override MetaDataCache<ContractIdState> ContractId => new StoreMetaDataCache<ContractIdState>(store, Prefixes.IX_ContractId);
 
         public ReadOnlyView(IReadOnlyStore store)

src/neo/Persistence/SnapshotView.cs

@@ -1,6 +1,7 @@
 using Neo.IO;
 using Neo.IO.Caching;
 using Neo.Ledger;
+using Neo.Network.P2P.Payloads;
 using System;
 
 namespace Neo.Persistence
@@ -17,8 +18,10 @@ public class SnapshotView : StoreView, IDisposable
         public override DataCache<UInt160, ContractState> Contracts { get; }
         public override DataCache<StorageKey, StorageItem> Storages { get; }
         public override DataCache<SerializableWrapper<uint>, HeaderHashList> HeaderHashList { get; }
+        public override DataCache<SerializableWrapper<uint>, HashIndexState> LocalStateRoot { get; }
         public override MetaDataCache<HashIndexState> BlockHashIndex { get; }
         public override MetaDataCache<HashIndexState> HeaderHashIndex { get; }
+        public override MetaDataCache<StateRoot> ValidatorsStateRoot { get; }
         public override MetaDataCache<ContractIdState> ContractId { get; }
 
         public SnapshotView(IStore store)
@@ -27,11 +30,13 @@ public SnapshotView(IStore store)
             Blocks = new StoreDataCache<UInt256, TrimmedBlock>(snapshot, Prefixes.DATA_Block);
             Transactions = new StoreDataCache<UInt256, TransactionState>(snapshot, Prefixes.DATA_Transaction);
             Contracts = new StoreDataCache<UInt160, ContractState>(snapshot, Prefixes.ST_Contract);
-            Storages = new StoreDataCache<StorageKey, StorageItem>(snapshot, Prefixes.ST_Storage);
             HeaderHashList = new StoreDataCache<SerializableWrapper<uint>, HeaderHashList>(snapshot, Prefixes.IX_HeaderHashList);
+            LocalStateRoot = new StoreDataCache<SerializableWrapper<uint>, HashIndexState>(snapshot, Prefixes.ST_LocalStateRoot);
             BlockHashIndex = new StoreMetaDataCache<HashIndexState>(snapshot, Prefixes.IX_CurrentBlock);
             HeaderHashIndex = new StoreMetaDataCache<HashIndexState>(snapshot, Prefixes.IX_CurrentHeader);
             ContractId = new StoreMetaDataCache<ContractIdState>(snapshot, Prefixes.IX_ContractId);
+            ValidatorsStateRoot = new StoreMetaDataCache<StateRoot>(snapshot, Prefixes.IX_ValidatorsStateRoot);
+            Storages = new MPTDataCache<StorageKey, StorageItem>(snapshot, CurrentStateRootHash);//Need BlockHashIndex and LocalStateRoot loaded.
         }
 
         public override void Commit()

src/neo/Persistence/StoreView.cs

@@ -16,13 +16,16 @@ public abstract class StoreView
         public abstract DataCache<UInt160, ContractState> Contracts { get; }
         public abstract DataCache<StorageKey, StorageItem> Storages { get; }
         public abstract DataCache<SerializableWrapper<uint>, HeaderHashList> HeaderHashList { get; }
+        public abstract DataCache<SerializableWrapper<uint>, HashIndexState> LocalStateRoot { get; }
         public abstract MetaDataCache<HashIndexState> BlockHashIndex { get; }
         public abstract MetaDataCache<HashIndexState> HeaderHashIndex { get; }
+        public abstract MetaDataCache<StateRoot> ValidatorsStateRoot { get; }
         public abstract MetaDataCache<ContractIdState> ContractId { get; }
 
         public uint Height => BlockHashIndex.Get().Index;
         public uint HeaderHeight => HeaderHashIndex.Get().Index;
         public UInt256 CurrentBlockHash => BlockHashIndex.Get().Hash;
+        public UInt256 CurrentStateRootHash => LocalStateRoot.TryGet(Height)?.Hash ?? UInt256.Zero;
         public UInt256 CurrentHeaderHash => HeaderHashIndex.Get().Hash;
 
         public StoreView Clone()
@@ -37,9 +40,11 @@ public virtual void Commit()
             Contracts.Commit();
             Storages.Commit();
             HeaderHashList.Commit();
+            LocalStateRoot.Commit();
             BlockHashIndex.Commit();
             HeaderHashIndex.Commit();
             ContractId.Commit();
+            ValidatorsStateRoot.Commit();
         }
 
         public bool ContainsBlock(UInt256 hash)

tests/neo.UnitTests/Consensus/UT_Consensus.cs

@@ -10,6 +10,7 @@
 using Neo.Ledger;
 using Neo.Network.P2P;
 using Neo.Network.P2P.Payloads;
+using Neo.Persistence;
 using Neo.SmartContract;
 using Neo.SmartContract.Native;
 using Neo.UnitTests.Cryptography;
@@ -276,6 +277,7 @@ public void ConsensusService_SingleNodeActors_OnStart_PrepReq_PrepResponses_Comm
             {
                 Value = mockContext.Object.Validators.ToByteArray()
             });
+            mockContext.Object.Snapshot.UpdateLocalStateRoot();
             mockContext.Object.Snapshot.Commit();
             // ===============================================================
 
@@ -408,6 +410,7 @@ public void ConsensusService_SingleNodeActors_OnStart_PrepReq_PrepResponses_Comm
             Console.WriteLine("mockContext Reset for returning Blockchain.Singleton snapshot to original state.");
             mockContext.Object.Reset(0);
             mockContext.Object.Snapshot.Storages.Delete(CreateStorageKeyForNativeNeo(14));
+            mockContext.Object.Snapshot.UpdateLocalStateRoot();
             mockContext.Object.Snapshot.Commit();
 
             Console.WriteLine("mockContext Reset.");