diff --git a/storage/statedb/database.go b/storage/statedb/database.go index f1573226de..60c5776736 100644 --- a/storage/statedb/database.go +++ b/storage/statedb/database.go @@ -87,9 +87,9 @@ type DatabaseReader interface { type Database struct { diskDB database.DBManager // Persistent storage for matured trie nodes - nodes map[common.Hash]*cachedNode // Data and references relationships of a trie node - oldest common.Hash // Oldest tracked node, flush-list head - newest common.Hash // Newest tracked node, flush-list tail + nodes map[common.ExtHash]*cachedNode // Data and references relationships of a trie node + oldest common.ExtHash // Oldest tracked node, flush-list head + newest common.ExtHash // Newest tracked node, flush-list tail preimages map[common.Hash][]byte // Preimages of nodes from the secure trie @@ -174,8 +174,8 @@ type cachedNode struct { parents uint64 // Number of live nodes referencing this one children map[common.ExtHash]uint64 // External children referenced by this node - flushPrev common.Hash // Previous node in the flush-list - flushNext common.Hash // Next node in the flush-list + flushPrev common.ExtHash // Previous node in the flush-list + flushNext common.ExtHash // Next node in the flush-list } // rlp returns the raw rlp encoded blob of the cached trie node, either directly @@ -313,7 +313,7 @@ func NewDatabaseWithNewCache(diskDB database.DBManager, cacheConfig *TrieNodeCac return &Database{ diskDB: diskDB, - nodes: map[common.Hash]*cachedNode{{}: {}}, + nodes: map[common.ExtHash]*cachedNode{{}: {}}, preimages: make(map[common.Hash][]byte), trieNodeCache: trieNodeCache, trieNodeCacheConfig: cacheConfig, @@ -326,7 +326,7 @@ func NewDatabaseWithNewCache(diskDB database.DBManager, cacheConfig *TrieNodeCac func NewDatabaseWithExistingCache(diskDB database.DBManager, cache TrieNodeCache) *Database { return &Database{ diskDB: diskDB, - nodes: map[common.Hash]*cachedNode{{}: {}}, + nodes: map[common.ExtHash]*cachedNode{{}: {}}, preimages: make(map[common.Hash][]byte), trieNodeCache: cache, } @@ -420,8 +420,7 @@ func (db *Database) NodeChildren(hash common.ExtHash) ([]common.ExtHash, error) // The blob size must be specified to allow proper size tracking. // All nodes inserted by this function will be reference tracked // and in theory should only used for **trie nodes** insertion. -func (db *Database) insert(_hash common.ExtHash, lenEncoded uint16, node node) { - hash := _hash.Unextend() +func (db *Database) insert(hash common.ExtHash, lenEncoded uint16, node node) { // If the node's already cached, skip if _, ok := db.nodes[hash]; ok { return @@ -433,20 +432,20 @@ func (db *Database) insert(_hash common.ExtHash, lenEncoded uint16, node node) { flushPrev: db.newest, } for _, child := range entry.childs() { - if c := db.nodes[child.Unextend()]; c != nil { + if c := db.nodes[child]; c != nil { c.parents++ } } db.nodes[hash] = entry // Update the flush-list endpoints - if db.oldest == (common.Hash{}) { + if db.oldest.IsEmpty() { db.oldest, db.newest = hash, hash } else { if _, ok := db.nodes[db.newest]; !ok { missingNewest := db.newest db.newest = db.getLastNodeHashInFlushList() - db.nodes[db.newest].flushNext = common.Hash{} + db.nodes[db.newest].flushNext = common.ExtHash{} logger.Error("Found a newest node for missingNewest", "oldNewest", missingNewest, "newNewest", db.newest) } db.nodes[db.newest].flushNext, db.newest = hash, hash @@ -489,11 +488,9 @@ func (db *Database) setCachedNode(hash common.ExtHash, enc []byte) { // node retrieves a cached trie node from memory, or returns nil if node can be // found in the memory cache. -func (db *Database) node(_hash common.ExtHash) (n node, fromDB bool) { - // TODO-Klaytn-Pruning: Use ExtHash in node() - hash := _hash.Unextend() +func (db *Database) node(hash common.ExtHash) (n node, fromDB bool) { // Retrieve the node from the trie node cache if available - if enc := db.getCachedNode(_hash); enc != nil { + if enc := db.getCachedNode(hash); enc != nil { if dec, err := decodeNode(hash[:], enc); err == nil { return dec, false } else { @@ -506,29 +503,26 @@ func (db *Database) node(_hash common.ExtHash) (n node, fromDB bool) { node := db.nodes[hash] db.lock.RUnlock() if node != nil { - return node.obj(_hash), false + return node.obj(hash), false } // Content unavailable in memory, attempt to retrieve from disk - // TODO-Klaytn-Pruning: node() takes ExtHash - enc, err := db.diskDB.ReadTrieNode(hash.ExtendLegacy()) + enc, err := db.diskDB.ReadTrieNode(hash) if err != nil || enc == nil { return nil, true } - db.setCachedNode(_hash, enc) + db.setCachedNode(hash, enc) return mustDecodeNode(hash[:], enc), true } // Node retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the persistent database for the content. -func (db *Database) Node(_hash common.ExtHash) ([]byte, error) { - // TODO-Klaytn-Pruning: Use ExtHash in Node() - hash := _hash.Unextend() - if _hash.IsEmpty() { +func (db *Database) Node(hash common.ExtHash) ([]byte, error) { + if hash.IsEmpty() { return nil, ErrZeroHashNode } // Retrieve the node from the trie node cache if available - if enc := db.getCachedNode(_hash); enc != nil { + if enc := db.getCachedNode(hash); enc != nil { return enc, nil } @@ -541,24 +535,21 @@ func (db *Database) Node(_hash common.ExtHash) ([]byte, error) { return node.rlp(), nil } // Content unavailable in memory, attempt to retrieve from disk - // TODO-Klaytn-Pruning: Node() takes ExtHash - enc, err := db.diskDB.ReadTrieNode(hash.ExtendLegacy()) + enc, err := db.diskDB.ReadTrieNode(hash) if err == nil && enc != nil { - db.setCachedNode(_hash, enc) + db.setCachedNode(hash, enc) } return enc, err } // NodeFromOld retrieves an encoded cached trie node from memory. If it cannot be found // cached, the method queries the old persistent database for the content. -func (db *Database) NodeFromOld(_hash common.ExtHash) ([]byte, error) { - // TODO-Klaytn-Pruning: Use ExtHash in NodeFromOld() - hash := _hash.Unextend() - if _hash.IsEmpty() { +func (db *Database) NodeFromOld(hash common.ExtHash) ([]byte, error) { + if hash.IsEmpty() { return nil, ErrZeroHashNode } // Retrieve the node from the trie node cache if available - if enc := db.getCachedNode(_hash); enc != nil { + if enc := db.getCachedNode(hash); enc != nil { return enc, nil } @@ -571,18 +562,15 @@ func (db *Database) NodeFromOld(_hash common.ExtHash) ([]byte, error) { return node.rlp(), nil } // Content unavailable in memory, attempt to retrieve from disk - // TODO-Klaytn-Pruning: NodeFromOld() takes ExtHash - enc, err := db.diskDB.ReadTrieNodeFromOld(hash.ExtendLegacy()) + enc, err := db.diskDB.ReadTrieNodeFromOld(hash) if err == nil && enc != nil { - db.setCachedNode(_hash, enc) + db.setCachedNode(hash, enc) } return enc, err } // DoesExistCachedNode returns if the node exists on cached trie node in memory. -func (db *Database) DoesExistCachedNode(_hash common.ExtHash) bool { - // TODO-Klaytn-Pruning: Use ExtHash in DoesExistCachedNode() - hash := _hash.Unextend() +func (db *Database) DoesExistCachedNode(hash common.ExtHash) bool { // Retrieve the node from cache if available db.lock.RLock() _, ok := db.nodes[hash] @@ -591,17 +579,14 @@ func (db *Database) DoesExistCachedNode(_hash common.ExtHash) bool { } // DoesExistNodeInPersistent returns if the node exists on the persistent database or its cache. -func (db *Database) DoesExistNodeInPersistent(_hash common.ExtHash) bool { - // TODO-Klaytn-Pruning: Use ExtHash in DoesExistCachedNode() - hash := _hash.Unextend() +func (db *Database) DoesExistNodeInPersistent(hash common.ExtHash) bool { // Retrieve the node from DB cache if available - if enc := db.getCachedNode(_hash); enc != nil { + if enc := db.getCachedNode(hash); enc != nil { return true } // Content unavailable in DB cache, attempt to retrieve from disk - // TODO-Klaytn-Pruning: DoesExistNodeInPersistent() takes ExtHash - enc, err := db.diskDB.ReadTrieNode(hash.ExtendLegacy()) + enc, err := db.diskDB.ReadTrieNode(hash) if err == nil && enc != nil { return true } @@ -633,8 +618,8 @@ func (db *Database) Nodes() []common.ExtHash { hashes := make([]common.ExtHash, 0, len(db.nodes)) for hash := range db.nodes { - if hash != (common.Hash{}) { // Special case for "root" references/nodes - hashes = append(hashes, hash.ExtendLegacy()) + if !hash.IsEmpty() { // Special case for "root" references/nodes + hashes = append(hashes, hash) } } return hashes @@ -666,9 +651,7 @@ func (db *Database) Reference(child common.ExtHash, parent common.ExtHash) { } // reference is the private locked version of Reference. -func (db *Database) reference(_child common.ExtHash, _parent common.ExtHash) { - child := _child.Unextend() - parent := _parent.Unextend() +func (db *Database) reference(child common.ExtHash, parent common.ExtHash) { // If the node does not exist, it's a node pulled from disk, skip node, ok := db.nodes[child] if !ok { @@ -677,11 +660,11 @@ func (db *Database) reference(_child common.ExtHash, _parent common.ExtHash) { // If the reference already exists, only duplicate for roots if db.nodes[parent].children == nil { db.nodes[parent].children = make(map[common.ExtHash]uint64) - } else if _, ok = db.nodes[parent].children[_child]; ok && parent != (common.Hash{}) { + } else if _, ok = db.nodes[parent].children[child]; ok && !parent.IsEmpty() { return } node.parents++ - db.nodes[parent].children[_child]++ + db.nodes[parent].children[child]++ } // DereferenceRoot removes an existing reference from a root node. @@ -716,16 +699,14 @@ func (db *Database) DereferenceRoot(root common.Hash) { } // dereference is the private locked version of Dereference. -func (db *Database) dereference(_child common.ExtHash, _parent common.ExtHash) { - child := _child.Unextend() - parent := _parent.Unextend() +func (db *Database) dereference(child common.ExtHash, parent common.ExtHash) { // Dereference the parent-child node := db.nodes[parent] - if node.children != nil && node.children[_child] > 0 { - node.children[_child]-- - if node.children[_child] == 0 { - delete(node.children, _child) + if node.children != nil && node.children[child] > 0 { + node.children[child]-- + if node.children[child] == 0 { + delete(node.children, child) } } // If the node does not exist, it's a previously committed node. @@ -746,7 +727,7 @@ func (db *Database) dereference(_child common.ExtHash, _parent common.ExtHash) { db.removeNodeInFlushList(child) // Dereference all children and delete the node for _, hash := range node.childs() { - db.dereference(hash, child.ExtendLegacy()) + db.dereference(hash, child) } delete(db.nodes, child) db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) @@ -782,17 +763,17 @@ func (db *Database) Cap(limit common.StorageSize) error { // Keep committing nodes from the flush-list until we're below allowance oldest := db.oldest batch := db.diskDB.NewBatch(database.StateTrieDB) - for size > limit && oldest != (common.Hash{}) { + for size > limit && !oldest.IsEmpty() { // Fetch the oldest referenced node and push into the batch node := db.nodes[oldest] enc := node.rlp() - db.diskDB.PutTrieNodeToBatch(batch, oldest.ExtendLegacy(), enc) + db.diskDB.PutTrieNodeToBatch(batch, oldest, enc) if _, err := database.WriteBatchesOverThreshold(batch); err != nil { db.lock.RUnlock() return err } - db.setCachedNode(oldest.ExtendLegacy(), enc) + db.setCachedNode(oldest, enc) // Iterate to the next flush item, or abort if the size cap was achieved. Size // is the total size, including both the useful cached data (hash -> blob), as // well as the flushlist metadata (2*hash). When flushing items from the cache, @@ -824,10 +805,10 @@ func (db *Database) Cap(limit common.StorageSize) error { db.nodesSize -= common.StorageSize(common.HashLength + int(node.size)) } - if db.oldest != (common.Hash{}) { - db.nodes[db.oldest].flushPrev = common.Hash{} + if !db.oldest.IsEmpty() { + db.nodes[db.oldest].flushPrev = common.ExtHash{} } else { - db.newest = common.Hash{} + db.newest = common.ExtHash{} } db.flushnodes += uint64(nodes - len(db.nodes)) db.flushsize += nodeSize - db.nodesSize @@ -858,7 +839,7 @@ type commitResult struct { } func (db *Database) writeBatchNodes(node common.ExtHash) error { - rootNode, ok := db.nodes[node.Unextend()] + rootNode, ok := db.nodes[node] if !ok { return nil } @@ -969,7 +950,7 @@ func (db *Database) CommitRoot(root common.Hash, report bool, blockNum uint64) e // commit iteratively encodes nodes from parents to child nodes. func (db *Database) commit(hash common.ExtHash, resultCh chan<- commitResult) { - node, ok := db.nodes[hash.Unextend()] + node, ok := db.nodes[hash] if !ok { return } @@ -986,8 +967,7 @@ func (db *Database) commit(hash common.ExtHash, resultCh chan<- commitResult) { // persisted trie is removed from the cache. The reason behind the two-phase // commit is to ensure consistent data availability while moving from memory // to disk. -func (db *Database) uncache(_hash common.ExtHash) { - hash := _hash.Unextend() +func (db *Database) uncache(hash common.ExtHash) { // If the node does not exists, we're done on this path node, ok := db.nodes[hash] if !ok { @@ -1026,13 +1006,13 @@ func (db *Database) verifyIntegrity() { // Iterate over all the cached nodes and accumulate them into a set reachable := map[common.ExtHash]struct{}{{}: {}} - for child := range db.nodes[common.Hash{}].children { + for child := range db.nodes[common.ExtHash{}].children { db.accumulate(child, reachable) } // Find any unreachable but cached nodes unreachable := []string{} for hash, node := range db.nodes { - if _, ok := reachable[hash.ExtendLegacy()]; !ok { + if _, ok := reachable[hash]; !ok { unreachable = append(unreachable, fmt.Sprintf("%x: {Node: %v, Parents: %d, Prev: %x, Next: %x}", hash, node.node, node.parents, node.flushPrev, node.flushNext)) } @@ -1046,7 +1026,7 @@ func (db *Database) verifyIntegrity() { // cached children found in memory. func (db *Database) accumulate(hash common.ExtHash, reachable map[common.ExtHash]struct{}) { // Mark the node reachable if present in the memory cache - node, ok := db.nodes[hash.Unextend()] + node, ok := db.nodes[hash] if !ok { return } @@ -1058,29 +1038,29 @@ func (db *Database) accumulate(hash common.ExtHash, reachable map[common.ExtHash } } -func (db *Database) removeNodeInFlushList(hash common.Hash) { +func (db *Database) removeNodeInFlushList(hash common.ExtHash) { node, ok := db.nodes[hash] if !ok { return } if hash == db.oldest && hash == db.newest { - db.oldest = common.Hash{} - db.newest = common.Hash{} + db.oldest = common.ExtHash{} + db.newest = common.ExtHash{} } else if hash == db.oldest { db.oldest = node.flushNext - db.nodes[node.flushNext].flushPrev = common.Hash{} + db.nodes[node.flushNext].flushPrev = common.ExtHash{} } else if hash == db.newest { db.newest = node.flushPrev - db.nodes[node.flushPrev].flushNext = common.Hash{} + db.nodes[node.flushPrev].flushNext = common.ExtHash{} } else { db.nodes[node.flushPrev].flushNext = node.flushNext db.nodes[node.flushNext].flushPrev = node.flushPrev } } -func (db *Database) getLastNodeHashInFlushList() common.Hash { - var lastNodeHash common.Hash +func (db *Database) getLastNodeHashInFlushList() common.ExtHash { + var lastNodeHash common.ExtHash nodeHash := db.oldest for { if _, ok := db.nodes[nodeHash]; ok { @@ -1090,7 +1070,7 @@ func (db *Database) getLastNodeHashInFlushList() common.Hash { break } - if db.nodes[nodeHash].flushNext != (common.Hash{}) { + if !db.nodes[nodeHash].flushNext.IsEmpty() { nodeHash = db.nodes[nodeHash].flushNext } else { logger.Debug("found last noode in map of flush list") diff --git a/storage/statedb/database_test.go b/storage/statedb/database_test.go index 5cd7e070b9..b3ae2e085e 100644 --- a/storage/statedb/database_test.go +++ b/storage/statedb/database_test.go @@ -43,8 +43,8 @@ func TestDatabase_Reference(t *testing.T) { child := &cachedNode{} parent := &cachedNode{} - db.nodes[childHash] = child - db.nodes[parentHash] = parent + db.nodes[childExtHash] = child + db.nodes[parentExtHash] = parent // Call Reference after updating db.nodes db.Reference(childExtHash, parentExtHash) @@ -71,8 +71,8 @@ func TestDatabase_DeReference(t *testing.T) { child := &cachedNode{} parent := &cachedNode{} - db.nodes[childHash] = child - db.nodes[parentHash] = parent + db.nodes[childExtHash] = child + db.nodes[parentExtHash] = parent db.Reference(childExtHash, parentExtHash) assert.Equal(t, 3, len(db.nodes)) @@ -99,8 +99,8 @@ func TestDatabase_Size(t *testing.T) { child := &cachedNode{} parent := &cachedNode{} - db.nodes[childHash] = child - db.nodes[parentHash] = parent + db.nodes[childExtHash] = child + db.nodes[parentExtHash] = parent db.Reference(childExtHash, parentExtHash) diff --git a/storage/statedb/hasher_test.go b/storage/statedb/hasher_test.go index f2ba42a292..dbb8c7e28a 100644 --- a/storage/statedb/hasher_test.go +++ b/storage/statedb/hasher_test.go @@ -38,7 +38,7 @@ func checkHasherHashFunc(t *testing.T, name string, tc testNodeEncodingTC, hashF assert.Equal(t, hashNode(tc.hash), cachedHash, name) hash := common.BytesToExtHash(tc.hash) - inserted := db.nodes[hash.Unextend()] + inserted := db.nodes[hash] require.NotNil(t, inserted) assert.Equal(t, tc.inserted, inserted.node, name) diff --git a/storage/statedb/iterator_test.go b/storage/statedb/iterator_test.go index 640250fa26..3ac01c8ce1 100644 --- a/storage/statedb/iterator_test.go +++ b/storage/statedb/iterator_test.go @@ -118,8 +118,8 @@ func TestNodeIteratorCoverage(t *testing.T) { } } for hash, obj := range db.nodes { - if obj != nil && hash != (common.Hash{}) { - if _, ok := hashes[hash]; !ok { + if obj != nil && !hash.IsEmpty() { + if _, ok := hashes[hash.Unextend()]; !ok { t.Errorf("state entry not reported %x", hash) } } @@ -343,8 +343,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { } if memonly { - robj = triedb.nodes[rkey] - delete(triedb.nodes, rkey) + robj = triedb.nodes[nodehash] + delete(triedb.nodes, nodehash) } else { rval, _ = dbm.ReadTrieNode(nodehash) dbm.DeleteTrieNode(nodehash) @@ -360,7 +360,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { // Add the node back and continue iteration. if memonly { - triedb.nodes[rkey] = robj + triedb.nodes[nodehash] = robj } else { dbm.WriteTrieNode(nodehash, rval) } @@ -405,8 +405,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { barNodeObj *cachedNode ) if memonly { - barNodeObj = triedb.nodes[barNodeHash] - delete(triedb.nodes, barNodeHash) + barNodeObj = triedb.nodes[nodehash] + delete(triedb.nodes, nodehash) } else { barNodeBlob, _ = dbm.ReadTrieNode(nodehash) dbm.DeleteTrieNode(nodehash) @@ -423,7 +423,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { } // Reinsert the missing node. if memonly { - triedb.nodes[barNodeHash] = barNodeObj + triedb.nodes[nodehash] = barNodeObj } else { dbm.WriteTrieNode(nodehash, barNodeBlob) } diff --git a/storage/statedb/trie_test.go b/storage/statedb/trie_test.go index d688e78393..6bfd8942fe 100644 --- a/storage/statedb/trie_test.go +++ b/storage/statedb/trie_test.go @@ -122,10 +122,9 @@ func testMissingNode(t *testing.T, memonly bool) { t.Errorf("Unexpected error: %v", err) } - hash := common.HexToHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9") - nodehash := hash.ExtendLegacy() + nodehash := common.HexToExtHash("0xe1d943cc8f061a0c0b98162830b970395ac9315654824bf21b73b891365262f9") if memonly { - delete(triedb.nodes, hash) + delete(triedb.nodes, nodehash) } else { dbm.DeleteTrieNode(nodehash) }