Upgrade rawdb and statedb codes to add the latest functionalities of …

…ethdb (#4374)

* added bloom filter

* upgrade rawdb and statedb

* change var name and remove extra comments

* return back fake storage in case if we need it for test later

* add the previous change back

* remove some extra entries from go.mod

* fix WritePreimages to use batch

* mark unused functions which are ported over from eth

---------

Co-authored-by: Casey Gardiner <117784577+ONECasey@users.noreply.github.com>

cmd/harmony/dumpdb.go

@@ -366,12 +366,12 @@ func (db *KakashiDB) stateDataDump(block *types.Block) {
 
 func dumpMain(srcDBDir, destDBDir string, batchLimit int) {
 	fmt.Println("===dumpMain===")
-	srcDB, err := rawdb.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
+	srcDB, err := ethRawDB.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
 	if err != nil {
 		fmt.Println("open src db error:", err)
 		os.Exit(-1)
 	}
-	destDB, err := rawdb.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
+	destDB, err := ethRawDB.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
 	if err != nil {
 		fmt.Println("open dest db error:", err)
 		os.Exit(-1)

core/rawdb/accessors_metadata.go

@@ -195,34 +195,3 @@ func WriteTransitionStatus(db ethdb.KeyValueWriter, data []byte) {
 		utils.Logger().Error().Err(err).Msg("Failed to store the eth2 transition status")
 	}
 }
-
-// WriteLeaderRotationMeta writes the leader continuous blocks count to the database.
-func WriteLeaderRotationMeta(db DatabaseWriter, leader []byte, epoch uint64, count, shifts uint64) error {
-	if len(leader) != bls.PublicKeySizeInBytes {
-		return errors.New("invalid leader public key size")
-	}
-	value := make([]byte, bls.PublicKeySizeInBytes+8*3)
-	copy(value, leader)
-	binary.LittleEndian.PutUint64(value[len(leader)+8*0:], epoch)
-	binary.LittleEndian.PutUint64(value[len(leader)+8*1:], count)
-	binary.LittleEndian.PutUint64(value[len(leader)+8*2:], shifts)
-	if err := db.Put(leaderContinuousBlocksCountKey(), value); err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to store leader continuous blocks count")
-		return err
-	}
-	return nil
-}
-
-// ReadLeaderRotationMeta retrieves the leader continuous blocks count from the database.
-func ReadLeaderRotationMeta(db DatabaseReader) (pubKeyBytes []byte, epoch, count, shifts uint64, err error) {
-	data, _ := db.Get(leaderContinuousBlocksCountKey())
-	if len(data) != bls.PublicKeySizeInBytes+24 {
-		return nil, 0, 0, 0, errors.New("invalid leader continuous blocks count")
-	}
-
-	pubKeyBytes = data[:bls.PublicKeySizeInBytes]
-	epoch = binary.LittleEndian.Uint64(data[bls.PublicKeySizeInBytes:])
-	count = binary.LittleEndian.Uint64(data[bls.PublicKeySizeInBytes+8:])
-	shifts = binary.LittleEndian.Uint64(data[bls.PublicKeySizeInBytes+16:])
-	return pubKeyBytes, epoch, count, shifts, nil
-}

core/rawdb/accessors_state.go

@@ -93,57 +93,3 @@ func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
 		utils.Logger().Error().Err(err).Msg("Failed to delete contract code")
 	}
 }
-
-// ReadValidatorCode retrieves the validator code of the provided code hash.
-func ReadValidatorCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
-	// Try with the prefixed code scheme first, if not then try with legacy
-	// scheme.
-	data := ReadValidatorCodeWithPrefix(db, hash)
-	if len(data) != 0 {
-		return data
-	}
-	data, _ = db.Get(hash.Bytes())
-	return data
-}
-
-// ReadValidatorCodeWithPrefix retrieves the validator code of the provided code hash.
-// The main difference between this function and ReadValidatorCode is this function
-// will only check the existence with latest scheme(with prefix).
-func ReadValidatorCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
-	data, _ := db.Get(validatorCodeKey(hash))
-	return data
-}
-
-// HasValidatorCode checks if the validator code corresponding to the
-// provided code hash is present in the db.
-func HasValidatorCode(db ethdb.KeyValueReader, hash common.Hash) bool {
-	// Try with the prefixed code scheme first, if not then try with legacy
-	// scheme.
-	if ok := HasValidatorCodeWithPrefix(db, hash); ok {
-		return true
-	}
-	ok, _ := db.Has(hash.Bytes())
-	return ok
-}
-
-// HasValidatorCodeWithPrefix checks if the validator code corresponding to the
-// provided code hash is present in the db. This function will only check
-// presence using the prefix-scheme.
-func HasValidatorCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool {
-	ok, _ := db.Has(validatorCodeKey(hash))
-	return ok
-}
-
-// WriteValidatorCode writes the provided validator code to database.
-func WriteValidatorCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
-	if err := db.Put(validatorCodeKey(hash), code); err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to store validator code")
-	}
-}
-
-// DeleteValidatorCode deletes the specified validator code from the database.
-func DeleteValidatorCode(db ethdb.KeyValueWriter, hash common.Hash) {
-	if err := db.Delete(validatorCodeKey(hash)); err != nil {
-		utils.Logger().Error().Err(err).Msg("Failed to delete validator code")
-	}
-}

core/rawdb/database.go

@@ -316,7 +316,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		hashNumPairings stat
 		tries           stat
 		codes           stat
-		validatorCodes  stat
 		txLookups       stat
 		accountSnaps    stat
 		storageSnaps    stat
@@ -360,8 +359,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			tries.Add(size)
 		case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
 			codes.Add(size)
-		case bytes.HasPrefix(key, ValidatorCodePrefix) && len(key) == len(ValidatorCodePrefix)+common.HashLength:
-			validatorCodes.Add(size)
 		case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
 			txLookups.Add(size)
 		case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
@@ -425,7 +422,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
 		{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
 		{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
-		{"Key-Value store", "Validator codes", validatorCodes.Size(), validatorCodes.Count()},
 		{"Key-Value store", "Trie nodes", tries.Size(), tries.Count()},
 		{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
 		{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},

core/rawdb/schema.go

@@ -126,11 +126,10 @@ var (
 	snapdbInfoKey = []byte("SnapdbInfo")
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
-	SnapshotAccountPrefix = []byte("a")  // SnapshotAccountPrefix + account hash -> account trie value
-	SnapshotStoragePrefix = []byte("o")  // SnapshotStoragePrefix + account hash + storage hash -> storage trie value
-	CodePrefix            = []byte("c")  // CodePrefix + code hash -> account code
-	ValidatorCodePrefix   = []byte("vc") // ValidatorCodePrefix + code hash -> validator code
-	skeletonHeaderPrefix  = []byte("S")  // skeletonHeaderPrefix + num (uint64 big endian) -> header
+	SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value
+	SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value
+	CodePrefix            = []byte("c") // CodePrefix + code hash -> account code -> We not using this at the moment
+	skeletonHeaderPrefix  = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header
 
 	// Path-based trie node scheme.
 	trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node
@@ -185,7 +184,8 @@ func skeletonHeaderKey(number uint64) []byte {
 
 // codeKey = CodePrefix + hash
 func codeKey(hash common.Hash) []byte {
-	return append(CodePrefix, hash.Bytes()...)
+	// We don't use any prefix for code key, otherwise we should return append(CodePrefix, hash.Bytes()...)
+	return hash.Bytes()
 }
 
 // IsCodeKey reports whether the given byte slice is the key of contract code,
@@ -197,20 +197,6 @@ func IsCodeKey(key []byte) (bool, []byte) {
 	return false, nil
 }
 
-// validatorCodeKey = ValidatorCodePrefix + hash
-func validatorCodeKey(hash common.Hash) []byte {
-	return append(ValidatorCodePrefix, hash.Bytes()...)
-}
-
-// IsValidatorCodeKey reports whether the given byte slice is the key of validator code,
-// if so return the raw code hash as well.
-func IsValidatorCodeKey(key []byte) (bool, []byte) {
-	if bytes.HasPrefix(key, ValidatorCodePrefix) && len(key) == common.HashLength+len(ValidatorCodePrefix) {
-		return true, key[len(ValidatorCodePrefix):]
-	}
-	return false, nil
-}
-
 // genesisStateSpecKey = genesisPrefix + hash
 func genesisStateSpecKey(hash common.Hash) []byte {
 	return append(genesisPrefix, hash.Bytes()...)

core/state/database.go

@@ -53,12 +53,6 @@ type Database interface {
 	// ContractCodeSize retrieves a particular contracts code's size.
 	ContractCodeSize(addrHash, codeHash common.Hash) (int, error)
 
-	// ValidatorCode retrieves a particular validator's code.
-	ValidatorCode(addrHash, codeHash common.Hash) ([]byte, error)
-
-	// ValidatorCodeSize retrieves a particular validator code's size.
-	ValidatorCodeSize(addrHash, codeHash common.Hash) (int, error)
-
 	// DiskDB returns the underlying key-value disk database.
 	DiskDB() ethdb.KeyValueStore
 
@@ -240,47 +234,6 @@ func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, erro
 	return len(code), err
 }
 
-// ValidatorCodeSize retrieves a particular validators code's size.
-func (db *cachingDB) ValidatorCodeSize(addrHash, codeHash common.Hash) (int, error) {
-	if cached, ok := db.codeSizeCache.Get(codeHash); ok {
-		return cached, nil
-	}
-	code, err := db.ValidatorCode(addrHash, codeHash)
-	return len(code), err
-}
-
-// ValidatorCode retrieves a particular validator's code.
-func (db *cachingDB) ValidatorCode(addrHash, codeHash common.Hash) ([]byte, error) {
-	code, _ := db.codeCache.Get(codeHash)
-	if len(code) > 0 {
-		return code, nil
-	}
-	code = rawdb.ReadValidatorCode(db.disk, codeHash)
-	if len(code) > 0 {
-		db.codeCache.Add(codeHash, code)
-		db.codeSizeCache.Add(codeHash, len(code))
-		return code, nil
-	}
-	return nil, errors.New("not found")
-}
-
-// ValidatorCodeWithPrefix retrieves a particular validator's code. If the
-// code can't be found in the cache, then check the existence with **new**
-// db scheme.
-func (db *cachingDB) ValidatorCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) {
-	code, _ := db.codeCache.Get(codeHash)
-	if len(code) > 0 {
-		return code, nil
-	}
-	code = rawdb.ReadValidatorCodeWithPrefix(db.disk, codeHash)
-	if len(code) > 0 {
-		db.codeCache.Add(codeHash, code)
-		db.codeSizeCache.Add(codeHash, len(code))
-		return code, nil
-	}
-	return nil, errors.New("not found")
-}
-
 // DiskDB returns the underlying key-value disk database.
 func (db *cachingDB) DiskDB() ethdb.KeyValueStore {
 	return db.disk

core/state/iterator.go

@@ -124,12 +124,6 @@ func (it *NodeIterator) step() error {
 		if err != nil {
 			return fmt.Errorf("code %x: %v", account.CodeHash, err)
 		}
-		if it.code == nil || len(it.code) == 0 {
-			it.code, err = it.state.db.ValidatorCode(addrHash, common.BytesToHash(account.CodeHash))
-			if err != nil {
-				return fmt.Errorf("code %x: %v", account.CodeHash, err)
-			}
-		}
 	}
 	it.accountHash = it.stateIt.Parent()
 	return nil

core/state/iterator_test.go

@@ -54,7 +54,7 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) {
 		acc.nonce = uint64(42 * i)
 
 		if i%3 == 0 {
-			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i}, false)
+			obj.SetCode(crypto.Keccak256Hash([]byte{i, i, i, i, i}), []byte{i, i, i, i, i})
 			acc.code = []byte{i, i, i, i, i}
 		}
 		if i%5 == 0 {

core/state/state_object.go

@@ -125,10 +125,10 @@ func newObject(db *DB, address common.Address, data types.StateAccount) *Object
 		data.Balance = new(big.Int)
 	}
 	if data.CodeHash == nil {
-		data.CodeHash = EmptyCodeHash.Bytes()
+		data.CodeHash = types.EmptyCodeHash.Bytes()
 	}
 	if data.Root == (common.Hash{}) {
-		data.Root = EmptyRootHash
+		data.Root = types.EmptyRootHash
 	}
 	return &Object{
 		db:             db,
@@ -175,7 +175,7 @@ func (s *Object) getTrie(db Database) (Trie, error) {
 	if s.trie == nil {
 		// Try fetching from prefetcher first
 		// We don't prefetch empty tries
-		if s.data.Root != EmptyRootHash && s.db.prefetcher != nil {
+		if s.data.Root != types.EmptyRootHash && s.db.prefetcher != nil {
 			// When the miner is creating the pending state, there is no
 			// prefetcher
 			s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root)
@@ -322,7 +322,7 @@ func (s *Object) finalise(prefetch bool) {
 			slotsToPrefetch = append(slotsToPrefetch, common.CopyBytes(key[:])) // Copy needed for closure
 		}
 	}
-	if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != EmptyRootHash {
+	if s.db.prefetcher != nil && prefetch && len(slotsToPrefetch) > 0 && s.data.Root != types.EmptyRootHash {
 		s.db.prefetcher.prefetch(s.addrHash, s.data.Root, slotsToPrefetch)
 	}
 	if len(s.dirtyStorage) > 0 {
@@ -481,18 +481,18 @@ func (s *Object) setBalance(amount *big.Int) {
 func (s *Object) ReturnGas(gas *big.Int) {}
 
 func (s *Object) deepCopy(db *DB) *Object {
-	stateObject := newObject(db, s.address, s.data)
+	Object := newObject(db, s.address, s.data)
 	if s.trie != nil {
-		stateObject.trie = db.db.CopyTrie(s.trie)
+		Object.trie = db.db.CopyTrie(s.trie)
 	}
-	stateObject.code = s.code
-	stateObject.dirtyStorage = s.dirtyStorage.Copy()
-	stateObject.originStorage = s.originStorage.Copy()
-	stateObject.pendingStorage = s.pendingStorage.Copy()
-	stateObject.suicided = s.suicided
-	stateObject.dirtyCode = s.dirtyCode
-	stateObject.deleted = s.deleted
-	return stateObject
+	Object.code = s.code
+	Object.dirtyStorage = s.dirtyStorage.Copy()
+	Object.originStorage = s.originStorage.Copy()
+	Object.pendingStorage = s.pendingStorage.Copy()
+	Object.suicided = s.suicided
+	Object.dirtyCode = s.dirtyCode
+	Object.deleted = s.deleted
+	return Object
 }
 
 //
@@ -509,7 +509,7 @@ func (s *Object) Code(db Database) []byte {
 	if s.code != nil {
 		return s.code
 	}
-	if bytes.Equal(s.CodeHash(), EmptyCodeHash.Bytes()) {
+	if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) {
 		return nil
 	}
 	var err error
@@ -538,43 +538,24 @@ func (s *Object) Code(db Database) []byte {
 	return code
 }
 
-// CodeSize returns the size of the contract/validator code associated with this object,
+// CodeSize returns the size of the contract code associated with this object,
 // or zero if none. This method is an almost mirror of Code, but uses a cache
 // inside the database to avoid loading codes seen recently.
 func (s *Object) CodeSize(db Database) int {
 	if s.code != nil {
 		return len(s.code)
 	}
-	if bytes.Equal(s.CodeHash(), EmptyCodeHash.Bytes()) {
+	if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) {
 		return 0
 	}
-	var err error
-	size := int(0)
-
-	// if it's not set for validator wrapper, then it may be either contract code or validator wrapper (old version of db
-	// don't have any prefix to differentiate between them)
-	// so, if it's not set for validator wrapper, we need to check contract code as well
-	if !s.validatorWrapper {
-		size, err = db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash()))
-	}
-	// if it couldn't get contract code or it is set to validator wrapper, then it tries to retrieve validator wrapper code
-	if s.validatorWrapper || err != nil {
-		vcSize, errVCSize := db.ValidatorCodeSize(s.addrHash, common.BytesToHash(s.CodeHash()))
-		if errVCSize == nil && vcSize > 0 {
-			return size
-		}
-		if s.validatorWrapper {
-			s.setError(fmt.Errorf("can't load validator code size %x for account address hash %x : %v", s.CodeHash(), s.addrHash, err))
-		} else {
-			s.setError(fmt.Errorf("can't load contract/validator code size %x for account address hash %x : contract code size error: %v, validator code size error: %v",
-				s.CodeHash(), s.addrHash, err, errVCSize))
-		}
-		s.setError(fmt.Errorf("can't load code size %x (validator wrapper: %t): %v", s.CodeHash(), s.validatorWrapper, err))
+	size, err := db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash()))
+	if err != nil {
+		s.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err))
 	}
 	return size
 }
 
-func (s *Object) SetCode(codeHash common.Hash, code []byte, isValidatorCode bool) {
+func (s *Object) SetCode(codeHash common.Hash, code []byte) {
 	prevcode := s.Code(s.db.db)
 	s.db.journal.append(codeChange{
 		account:  &s.address,