Merge pull request #1737 from ElrondNetwork/EN-6407-trie-package-unit…

…-tests

En 6407 trie package unit tests

cmd/node/config/config.toml

@@ -187,7 +187,7 @@
      MaxOpenFiles = 10
 
 [TrieStorageManagerConfig]
-    PruningBufferLen = 1000
+    PruningBufferLen = 100000
     SnapshotsBufferLen = 1000000
     MaxSnapshots = 2
 
@@ -397,6 +397,8 @@
     CheckpointRoundsModulus = 100
     AccountsStatePruningEnabled = true
     PeerStatePruningEnabled = true
+    MaxStateTrieLevelInMemory = 5
+    MaxPeerTrieLevelInMemory = 5
 
 [BlockSizeThrottleConfig]
     MinSizeInBytes = 104857 # 104857 is 10% from 1MB

config/config.go

@@ -189,6 +189,8 @@ type StateTriesConfig struct {
 	CheckpointRoundsModulus     uint
 	AccountsStatePruningEnabled bool
 	PeerStatePruningEnabled     bool
+	MaxStateTrieLevelInMemory   uint
+	MaxPeerTrieLevelInMemory    uint
 }
 
 // TrieStorageManagerConfig will hold config information about trie storage manager

data/interface.go

@@ -210,7 +210,7 @@ type StorageManager interface {
 
 // TrieFactory creates new tries
 type TrieFactory interface {
-	Create(config.StorageConfig, string, bool) (StorageManager, Trie, error)
+	Create(config.StorageConfig, string, bool, uint) (StorageManager, Trie, error)
 	IsInterfaceNil() bool
 }
 

data/state/accountsDB_test.go

@@ -810,7 +810,8 @@ func TestAccountsDB_RevertToSnapshotShouldWork(t *testing.T) {
 	hsh := mock.HasherMock{}
 	accFactory := factory.NewAccountCreator()
 	storageManager, _ := trie.NewTrieStorageManagerWithoutPruning(mock.NewMemDbMock())
-	tr, _ := trie.NewTrie(storageManager, marsh, hsh)
+	maxTrieLevelInMemory := uint(5)
+	tr, _ := trie.NewTrie(storageManager, marsh, hsh, maxTrieLevelInMemory)
 
 	adb, _ := state.NewAccountsDB(tr, hsh, marsh, accFactory)
 

data/syncer/baseAccountsSyncer.go

@@ -16,29 +16,31 @@ import (
 )
 
 type baseAccountsSyncer struct {
-	hasher             hashing.Hasher
-	marshalizer        marshal.Marshalizer
-	trieSyncers        map[string]data.TrieSyncer
-	dataTries          map[string]data.Trie
-	mutex              sync.Mutex
-	trieStorageManager data.StorageManager
-	requestHandler     trie.RequestHandler
-	waitTime           time.Duration
-	shardId            uint32
-	cacher             storage.Cacher
-	rootHash           []byte
+	hasher               hashing.Hasher
+	marshalizer          marshal.Marshalizer
+	trieSyncers          map[string]data.TrieSyncer
+	dataTries            map[string]data.Trie
+	mutex                sync.Mutex
+	trieStorageManager   data.StorageManager
+	requestHandler       trie.RequestHandler
+	waitTime             time.Duration
+	shardId              uint32
+	cacher               storage.Cacher
+	rootHash             []byte
+	maxTrieLevelInMemory uint
 }
 
 const minWaitTime = time.Second
 
 // ArgsNewBaseAccountsSyncer defines the arguments needed for the new account syncer
 type ArgsNewBaseAccountsSyncer struct {
-	Hasher             hashing.Hasher
-	Marshalizer        marshal.Marshalizer
-	TrieStorageManager data.StorageManager
-	RequestHandler     trie.RequestHandler
-	WaitTime           time.Duration
-	Cacher             storage.Cacher
+	Hasher               hashing.Hasher
+	Marshalizer          marshal.Marshalizer
+	TrieStorageManager   data.StorageManager
+	RequestHandler       trie.RequestHandler
+	WaitTime             time.Duration
+	Cacher               storage.Cacher
+	MaxTrieLevelInMemory uint
 }
 
 func checkArgs(args ArgsNewBaseAccountsSyncer) error {
@@ -67,7 +69,7 @@ func checkArgs(args ArgsNewBaseAccountsSyncer) error {
 func (b *baseAccountsSyncer) syncMainTrie(rootHash []byte, trieTopic string, ctx context.Context) error {
 	b.rootHash = rootHash
 
-	dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher)
+	dataTrie, err := trie.NewTrie(b.trieStorageManager, b.marshalizer, b.hasher, b.maxTrieLevelInMemory)
 	if err != nil {
 		return err
 	}

data/syncer/userAccountsSyncer.go

@@ -81,7 +81,7 @@ func (u *userAccountsSyncer) SyncAccounts(rootHash []byte) error {
 
 func (u *userAccountsSyncer) syncAccountDataTries(rootHashes [][]byte, ctx context.Context) error {
 	for _, rootHash := range rootHashes {
-		dataTrie, err := trie.NewTrie(u.trieStorageManager, u.marshalizer, u.hasher)
+		dataTrie, err := trie.NewTrie(u.trieStorageManager, u.marshalizer, u.hasher, u.maxTrieLevelInMemory)
 		if err != nil {
 			return err
 		}

data/trie/branchNode.go

@@ -240,7 +240,7 @@ func (bn *branchNode) hashNode() ([]byte, error) {
 	return encodeNodeAndGetHash(bn)
 }
 
-func (bn *branchNode) commit(force bool, level byte, originDb data.DBWriteCacher, targetDb data.DBWriteCacher) error {
+func (bn *branchNode) commit(force bool, level byte, maxTrieLevelInMemory uint, originDb data.DBWriteCacher, targetDb data.DBWriteCacher) error {
 	level++
 	err := bn.isEmptyOrNil()
 	if err != nil {
@@ -264,7 +264,7 @@ func (bn *branchNode) commit(force bool, level byte, originDb data.DBWriteCacher
 			continue
 		}
 
-		err = bn.children[i].commit(force, level, originDb, targetDb)
+		err = bn.children[i].commit(force, level, maxTrieLevelInMemory, originDb, targetDb)
 		if err != nil {
 			return err
 		}
@@ -274,7 +274,7 @@ func (bn *branchNode) commit(force bool, level byte, originDb data.DBWriteCacher
 	if err != nil {
 		return err
 	}
-	if level == maxTrieLevelAfterCommit {
+	if uint(level) == maxTrieLevelInMemory {
 		var collapsed node
 		collapsed, err = bn.getCollapsed()
 		if err != nil {

data/trie/branchNode_test.go

@@ -1,6 +1,7 @@
 package trie
 
 import (
+	"bytes"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -187,9 +188,10 @@ func TestBranchNode_setRootHash(t *testing.T) {
 	marsh, hsh := getTestMarshAndHasher()
 	trieStorage1, _ := NewTrieStorageManager(db, marsh, hsh, cfg, &mock.EvictionWaitingList{}, config.TrieStorageManagerConfig{})
 	trieStorage2, _ := NewTrieStorageManager(db, marsh, hsh, cfg, &mock.EvictionWaitingList{}, config.TrieStorageManagerConfig{})
+	maxTrieLevelInMemory := uint(5)
 
-	tr1, _ := NewTrie(trieStorage1, marsh, hsh)
-	tr2, _ := NewTrie(trieStorage2, marsh, hsh)
+	tr1, _ := NewTrie(trieStorage1, marsh, hsh, maxTrieLevelInMemory)
+	tr2, _ := NewTrie(trieStorage2, marsh, hsh, maxTrieLevelInMemory)
 
 	maxIterations := 10000
 	for i := 0; i < maxIterations; i++ {
@@ -348,7 +350,7 @@ func TestBranchNode_commit(t *testing.T) {
 	hash, _ := encodeNodeAndGetHash(collapsedBn)
 	_ = bn.setHash()
 
-	err := bn.commit(false, 0, db, db)
+	err := bn.commit(false, 0, 5, db, db)
 	assert.Nil(t, err)
 
 	encNode, _ := db.Get(hash)
@@ -363,7 +365,7 @@ func TestBranchNode_commitEmptyNode(t *testing.T) {
 
 	bn := emptyDirtyBranchNode()
 
-	err := bn.commit(false, 0, nil, nil)
+	err := bn.commit(false, 0, 5, nil, nil)
 	assert.True(t, errors.Is(err, ErrEmptyBranchNode))
 }
 
@@ -372,7 +374,7 @@ func TestBranchNode_commitNilNode(t *testing.T) {
 
 	var bn *branchNode
 
-	err := bn.commit(false, 0, nil, nil)
+	err := bn.commit(false, 0, 5, nil, nil)
 	assert.True(t, errors.Is(err, ErrNilBranchNode))
 }
 
@@ -417,7 +419,7 @@ func TestBranchNode_resolveCollapsed(t *testing.T) {
 	childPos := byte(2)
 
 	_ = bn.setHash()
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 	resolved, _ := newLeafNode([]byte("dog"), []byte("dog"), bn.marsh, bn.hasher)
 	resolved.dirty = false
 	resolved.hash = bn.EncodedChildren[childPos]
@@ -518,7 +520,7 @@ func TestBranchNode_tryGetCollapsedNode(t *testing.T) {
 	bn, collapsedBn := getBnAndCollapsedBn(getTestMarshAndHasher())
 
 	_ = bn.setHash()
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 
 	childPos := byte(2)
 	key := append([]byte{childPos}, []byte("dog")...)
@@ -644,7 +646,7 @@ func TestBranchNode_insertCollapsedNode(t *testing.T) {
 	node, _ := newLeafNode(key, []byte("dogs"), bn.marsh, bn.hasher)
 
 	_ = bn.setHash()
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 
 	dirty, newBn, _, err := collapsedBn.insert(node, db)
 	assert.True(t, dirty)
@@ -663,7 +665,7 @@ func TestBranchNode_insertInStoredBnOnExistingPos(t *testing.T) {
 	key := append([]byte{childPos}, []byte("dog")...)
 	node, _ := newLeafNode(key, []byte("dogs"), bn.marsh, bn.hasher)
 
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 	bnHash := bn.getHash()
 	ln, _, _ := bn.getNext(key, db)
 	lnHash := ln.getHash()
@@ -684,7 +686,7 @@ func TestBranchNode_insertInStoredBnOnNilPos(t *testing.T) {
 	key := append([]byte{nilChildPos}, []byte("dog")...)
 	node, _ := newLeafNode(key, []byte("dogs"), bn.marsh, bn.hasher)
 
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 	bnHash := bn.getHash()
 	expectedHashes := [][]byte{bnHash}
 
@@ -763,7 +765,7 @@ func TestBranchNode_deleteFromStoredBn(t *testing.T) {
 	childPos := byte(2)
 	lnKey := append([]byte{childPos}, []byte("dog")...)
 
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 	bnHash := bn.getHash()
 	ln, _, _ := bn.getNext(lnKey, db)
 	lnHash := ln.getHash()
@@ -845,7 +847,7 @@ func TestBranchNode_deleteCollapsedNode(t *testing.T) {
 	db := mock.NewMemDbMock()
 	bn, collapsedBn := getBnAndCollapsedBn(getTestMarshAndHasher())
 	_ = bn.setHash()
-	_ = bn.commit(false, 0, db, db)
+	_ = bn.commit(false, 0, 5, db, db)
 
 	childPos := byte(2)
 	key := append([]byte{childPos}, []byte("dog")...)
@@ -1017,7 +1019,7 @@ func TestBranchNode_getChildrenCollapsedBn(t *testing.T) {
 
 	db := mock.NewMemDbMock()
 	bn, collapsedBn := getBnAndCollapsedBn(getTestMarshAndHasher())
-	_ = bn.commit(true, 0, db, db)
+	_ = bn.commit(true, 0, 5, db, db)
 
 	children, err := collapsedBn.getChildren(db)
 	assert.Nil(t, err)
@@ -1208,3 +1210,128 @@ func BenchmarkMarshallNodeJson(b *testing.B) {
 		_, _ = marsh.Marshal(bn)
 	}
 }
+
+func TestBranchNode_newBranchNodeNilMarshalizerShouldErr(t *testing.T) {
+	t.Parallel()
+
+	bn, err := newBranchNode(nil, mock.HasherMock{})
+	assert.Nil(t, bn)
+	assert.Equal(t, ErrNilMarshalizer, err)
+}
+
+func TestBranchNode_newBranchNodeNilHasherShouldErr(t *testing.T) {
+	t.Parallel()
+
+	bn, err := newBranchNode(&mock.MarshalizerMock{}, nil)
+	assert.Nil(t, bn)
+	assert.Equal(t, ErrNilHasher, err)
+}
+
+func TestBranchNode_newBranchNodeOkVals(t *testing.T) {
+	t.Parallel()
+
+	var children [nrOfChildren]node
+	marsh, hasher := getTestMarshAndHasher()
+	bn, err := newBranchNode(marsh, hasher)
+
+	assert.Nil(t, err)
+	assert.Equal(t, make([][]byte, nrOfChildren), bn.EncodedChildren)
+	assert.Equal(t, children, bn.children)
+	assert.Equal(t, marsh, bn.marsh)
+	assert.Equal(t, hasher, bn.hasher)
+	assert.True(t, bn.dirty)
+}
+
+func TestBranchNode_getMarshalizer(t *testing.T) {
+	t.Parallel()
+
+	expectedMarsh := &mock.MarshalizerMock{}
+	bn := &branchNode{
+		baseNode: &baseNode{
+			marsh: expectedMarsh,
+		},
+	}
+
+	marsh := bn.getMarshalizer()
+	assert.Equal(t, expectedMarsh, marsh)
+}
+
+func TestBranchNode_setRootHashCollapsedChildren(t *testing.T) {
+	t.Parallel()
+
+	marsh, hasher := getTestMarshAndHasher()
+	bn := &branchNode{
+		baseNode: &baseNode{
+			marsh:  marsh,
+			hasher: hasher,
+		},
+	}
+
+	_, collapsedBn := getBnAndCollapsedBn(marsh, hasher)
+	_, collapsedEn := getEnAndCollapsedEn()
+	collapsedLn := getLn(marsh, hasher)
+
+	bn.children[0] = collapsedBn
+	bn.children[1] = collapsedEn
+	bn.children[2] = collapsedLn
+
+	err := bn.setRootHash()
+	assert.Nil(t, err)
+}
+
+func TestBranchNode_commitCollapsesTrieIfMaxTrieLevelInMemoryIsReached(t *testing.T) {
+	t.Parallel()
+
+	bn, collapsedBn := getBnAndCollapsedBn(getTestMarshAndHasher())
+	_ = collapsedBn.setRootHash()
+
+	err := bn.commit(true, 0, 1, mock.NewMemDbMock(), mock.NewMemDbMock())
+	assert.Nil(t, err)
+
+	assert.Equal(t, collapsedBn.EncodedChildren, bn.EncodedChildren)
+	assert.Equal(t, collapsedBn.children, bn.children)
+	assert.Equal(t, collapsedBn.hash, bn.hash)
+}
+
+func TestBranchNode_reduceNodeBnChild(t *testing.T) {
+	t.Parallel()
+
+	marsh, hasher := getTestMarshAndHasher()
+	en, _ := getEnAndCollapsedEn()
+	pos := 5
+	expectedNode, _ := newExtensionNode([]byte{byte(pos)}, en.child, marsh, hasher)
+
+	newNode, err := en.child.reduceNode(pos)
+	assert.Nil(t, err)
+	assert.Equal(t, expectedNode, newNode)
+}
+
+func TestBranchNode_printShouldNotPanicEvenIfNodeIsCollapsed(t *testing.T) {
+	t.Parallel()
+
+	bnWriter := bytes.NewBuffer(make([]byte, 0))
+	collapsedBnWriter := bytes.NewBuffer(make([]byte, 0))
+
+	db := mock.NewMemDbMock()
+	bn, collapsedBn := getBnAndCollapsedBn(getTestMarshAndHasher())
+	_ = bn.commit(true, 0, 5, db, db)
+	_ = collapsedBn.commit(true, 0, 5, db, db)
+
+	bn.print(bnWriter, 0, db)
+	collapsedBn.print(collapsedBnWriter, 0, db)
+
+	assert.Equal(t, bnWriter.Bytes(), collapsedBnWriter.Bytes())
+}
+
+func TestBranchNode_getDirtyHashesFromCleanNode(t *testing.T) {
+	t.Parallel()
+
+	db := mock.NewMemDbMock()
+	bn, _ := getBnAndCollapsedBn(getTestMarshAndHasher())
+	_ = bn.commit(true, 0, 5, db, db)
+	dirtyHashes := make(data.ModifiedHashes)
+
+	err := bn.getDirtyHashes(dirtyHashes)
+	assert.Nil(t, err)
+	assert.Equal(t, 0, len(dirtyHashes))
+}