Upgrade rawdb and statedb codes to add the latest functionalities of ethdb

.gitignore

@@ -96,4 +96,5 @@ explorer_storage_*
 profiles/*.pb.gz
 
 # cache db
-cache_*_db
+cache_*_db
+cache

accounts/abi/bind/util_test.go

@@ -56,14 +56,17 @@ func TestWaitDeployed(t *testing.T) {
 	for name, test := range waitDeployedTests {
 		backend := backends.NewSimulatedBackend(
 			core.GenesisAlloc{
-				crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
+				crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)},
 			},
 			10000000,
 		)
 		defer backend.Close()
 
-		// Create the transaction.
-		tx := types.NewContractCreation(0, big.NewInt(0), test.gas, big.NewInt(1), common.FromHex(test.code))
+		// Create the transaction
+		head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough
+		gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1))
+
+		tx := types.NewContractCreation(0, big.NewInt(0), test.gas, gasPrice, common.FromHex(test.code))
 		tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey)
 
 		// Wait for it to get mined in the background.
@@ -99,15 +102,18 @@ func TestWaitDeployed(t *testing.T) {
 func TestWaitDeployedCornerCases(t *testing.T) {
 	backend := backends.NewSimulatedBackend(
 		core.GenesisAlloc{
-			crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000)},
+			crypto.PubkeyToAddress(testKey.PublicKey): {Balance: big.NewInt(10000000000000000)},
 		},
 		10000000,
 	)
 	defer backend.Close()
 
+	head, _ := backend.HeaderByNumber(context.Background(), nil) // Should be child's, good enough
+	gasPrice := new(big.Int).Add(head.BaseFee, big.NewInt(1))
+
 	// Create a transaction to an account.
 	code := "6060604052600a8060106000396000f360606040526008565b00"
-	tx := types.NewTransaction(0, common.HexToAddress("0x01"), big.NewInt(0), 3000000, big.NewInt(1), common.FromHex(code))
+	tx := types.NewTransaction(0, common.HexToAddress("0x01"), big.NewInt(0), 3000000, gasPrice, common.FromHex(code))
 	tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -119,7 +125,7 @@ func TestWaitDeployedCornerCases(t *testing.T) {
 	}
 
 	// Create a transaction that is not mined.
-	tx = types.NewContractCreation(1, big.NewInt(0), 3000000, big.NewInt(1), common.FromHex(code))
+	tx = types.NewContractCreation(1, big.NewInt(0), 3000000, gasPrice, common.FromHex(code))
 	tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey)
 
 	go func() {

accounts/external/backend.go

@@ -26,7 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
-	"github.com/ethereum/go-ethereum/signer/core"
+	"github.com/ethereum/go-ethereum/signer/core/apitypes"
 	"github.com/harmony-one/harmony/accounts"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/eth/rpc"
@@ -218,12 +218,13 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio
 		t := common.NewMixedcaseAddress(*tx.To())
 		to = &t
 	}
-	args := &core.SendTxArgs{
+	gas := hexutil.Big(*tx.GasPrice())
+	args := &apitypes.SendTxArgs{
 		Data:     &data,
 		Nonce:    hexutil.Uint64(tx.Nonce()),
 		Value:    hexutil.Big(*tx.Value()),
 		Gas:      hexutil.Uint64(tx.GasLimit()),
-		GasPrice: hexutil.Big(*tx.GasPrice()),
+		GasPrice: &gas,
 		To:       to,
 		From:     common.NewMixedcaseAddress(account.Address),
 	}

api/service/explorer/interface.go

@@ -42,7 +42,7 @@ type explorerDB struct {
 
 // newExplorerLvlDB new explorer storage using leveldb
 func newExplorerLvlDB(dbPath string) (database, error) {
-	db, err := leveldb.New(dbPath, 16, 500, "explorer_db")
+	db, err := leveldb.New(dbPath, 16, 500, "explorer_db", false)
 	if err != nil {
 		return nil, err
 	}
@@ -80,7 +80,7 @@ func (db *explorerDB) NewBatch() batch {
 }
 
 func (db *explorerDB) NewPrefixIterator(prefix []byte) iterator {
-	it := db.db.NewIteratorWithPrefix(prefix)
+	it := db.db.NewIterator(prefix, nil)
 	return it
 }
 
@@ -95,7 +95,7 @@ type sizedIterator struct {
 }
 
 func (db *explorerDB) newSizedIterator(start []byte, size int) *sizedIterator {
-	it := db.db.NewIteratorWithStart(start)
+	it := db.db.NewIterator(nil, start)
 	return &sizedIterator{
 		it:        it,
 		curIndex:  0,

cmd/harmony/dumpdb.go

@@ -176,7 +176,8 @@ func (db *KakashiDB) Close() error {
 	return db.Database.Close()
 }
 
-func (db *KakashiDB) OnRoot(common.Hash) {}
+func (db *KakashiDB) OnRoot(common.Hash)                          {}
+func (db *KakashiDB) OnAccount(common.Address, state.DumpAccount) {}
 
 // OnAccount implements DumpCollector interface
 func (db *KakashiDB) OnAccountStart(addr common.Address, acc state.DumpAccount) {
@@ -345,7 +346,7 @@ func (db *KakashiDB) stateDataDump(block *types.Block) {
 	fmt.Println("stateDataDump:", snapdbInfo.LastAccountKey.String(), snapdbInfo.LastAccountStateKey.String())
 	stateDB0 := state.NewDatabaseWithCache(db, STATEDB_CACHE_SIZE)
 	rootHash := block.Root()
-	stateDB, err := state.New(rootHash, stateDB0)
+	stateDB, err := state.New(rootHash, stateDB0, nil)
 	if err != nil {
 		panic(err)
 	}
@@ -354,8 +355,8 @@ func (db *KakashiDB) stateDataDump(block *types.Block) {
 	if len(snapdbInfo.LastAccountStateKey) > 0 {
 		stateKey := new(big.Int).SetBytes(snapdbInfo.LastAccountStateKey)
 		stateKey.Add(stateKey, big.NewInt(1))
-		config.StateStart = stateKey.Bytes()
-		if len(config.StateStart) != len(snapdbInfo.LastAccountStateKey) {
+		config.Start = stateKey.Bytes()
+		if len(config.Start) != len(snapdbInfo.LastAccountStateKey) {
 			panic("statekey overflow")
 		}
 	}
@@ -366,12 +367,12 @@ func (db *KakashiDB) stateDataDump(block *types.Block) {
 
 func dumpMain(srcDBDir, destDBDir string, batchLimit int) {
 	fmt.Println("===dumpMain===")
-	srcDB, err := ethRawDB.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "")
+	srcDB, err := ethRawDB.NewLevelDBDatabase(srcDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
 	if err != nil {
 		fmt.Println("open src db error:", err)
 		os.Exit(-1)
 	}
-	destDB, err := ethRawDB.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "")
+	destDB, err := ethRawDB.NewLevelDBDatabase(destDBDir, LEVELDB_CACHE_SIZE, LEVELDB_HANDLES, "", false)
 	if err != nil {
 		fmt.Println("open dest db error:", err)
 		os.Exit(-1)

core/blockchain_impl.go

@@ -135,9 +135,9 @@ type BlockChainImpl struct {
 	pruneBeaconChainEnable bool                // pruneBeaconChainEnable is enable prune BeaconChain feature
 	shardID                uint32              // Shard number
 
-	db     ethdb.Database // Low level persistent database to store final content in
-	triegc *prque.Prque   // Priority queue mapping block numbers to tries to gc
-	gcproc time.Duration  // Accumulates canonical block processing for trie dumping
+	db     ethdb.Database                   // Low level persistent database to store final content in
+	triegc *prque.Prque[int64, common.Hash] // Priority queue mapping block numbers to tries to gc
+	gcproc time.Duration                    // Accumulates canonical block processing for trie dumping
 
 	// The following two variables are used to clean up the cache of redis in tikv mode.
 	// This can improve the cache hit rate of redis
@@ -246,7 +246,7 @@ func newBlockChainWithOptions(
 		chainConfig:                   chainConfig,
 		cacheConfig:                   cacheConfig,
 		db:                            db,
-		triegc:                        prque.New(nil),
+		triegc:                        prque.New[int64, common.Hash](nil),
 		stateCache:                    stateCache,
 		quit:                          make(chan struct{}),
 		bodyCache:                     bodyCache,
@@ -462,7 +462,7 @@ func (bc *BlockChainImpl) ValidateNewBlock(block *types.Block, beaconChain Block
 }
 
 func (bc *BlockChainImpl) validateNewBlock(block *types.Block) error {
-	state, err := state.New(bc.CurrentBlock().Root(), bc.stateCache)
+	state, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, nil)
 	if err != nil {
 		return err
 	}
@@ -520,7 +520,7 @@ func (bc *BlockChainImpl) loadLastState() error {
 		return bc.Reset()
 	}
 	// Make sure the state associated with the block is available
-	if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
+	if _, err := state.New(currentBlock.Root(), bc.stateCache, nil); err != nil {
 		// Dangling block without a state associated, init from scratch
 		utils.Logger().Warn().
 			Str("number", currentBlock.Number().String()).
@@ -617,7 +617,7 @@ func (bc *BlockChainImpl) SetHead(head uint64) error {
 		headBlockGauge.Update(int64(newHeadBlock.NumberU64()))
 	}
 	if currentBlock := bc.CurrentBlock(); currentBlock != nil {
-		if _, err := state.New(currentBlock.Root(), bc.stateCache); err != nil {
+		if _, err := state.New(currentBlock.Root(), bc.stateCache, nil); err != nil {
 			// Rewound state missing, rolled back to before pivot, reset to genesis
 			bc.currentBlock.Store(bc.genesisBlock)
 			headBlockGauge.Update(int64(bc.genesisBlock.NumberU64()))
@@ -694,7 +694,7 @@ func (bc *BlockChainImpl) State() (*state.DB, error) {
 }
 
 func (bc *BlockChainImpl) StateAt(root common.Hash) (*state.DB, error) {
-	return state.New(root, bc.stateCache)
+	return state.New(root, bc.stateCache, nil)
 }
 
 func (bc *BlockChainImpl) Reset() error {
@@ -739,7 +739,7 @@ func (bc *BlockChainImpl) repair(head **types.Block) error {
 	valsToRemove := map[common.Address]struct{}{}
 	for {
 		// Abort if we've rewound to a head block that does have associated state
-		if _, err := state.New((*head).Root(), bc.stateCache); err == nil {
+		if _, err := state.New((*head).Root(), bc.stateCache, nil); err == nil {
 			utils.Logger().Info().
 				Str("number", (*head).Number().String()).
 				Str("hash", (*head).Hash().Hex()).
@@ -1007,7 +1007,7 @@ func (bc *BlockChainImpl) GetReceiptsByHash(hash common.Hash) types.Receipts {
 		return nil
 	}
 
-	receipts := rawdb.ReadReceipts(bc.db, hash, *number)
+	receipts := rawdb.ReadReceipts(bc.db, hash, *number, nil)
 	bc.receiptsCache.Add(hash, receipts)
 	return receipts
 }
@@ -1091,7 +1091,8 @@ func (bc *BlockChainImpl) Stop() {
 			}
 		}
 		for !bc.triegc.Empty() {
-			triedb.Dereference(bc.triegc.PopItem().(common.Hash))
+			v := common.Hash(bc.triegc.PopItem())
+			triedb.Dereference(v)
 		}
 		if size, _ := triedb.Size(); size != 0 {
 			utils.Logger().Error().Msg("Dangling trie nodes after full cleanup")
@@ -1400,6 +1401,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 	} else {
 		// Full but not archive node, do proper garbage collection
 		triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
+		// r := common.Hash(root)
 		bc.triegc.Push(root, -int64(block.NumberU64()))
 
 		if current := block.NumberU64(); current > bc.cacheConfig.TriesInMemory {
@@ -1442,7 +1444,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 					if -number > bc.maxGarbCollectedBlkNum {
 						bc.maxGarbCollectedBlkNum = -number
 					}
-					triedb.Dereference(root.(common.Hash))
+					triedb.Dereference(root)
 				}
 			}
 		}
@@ -1473,7 +1475,7 @@ func (bc *BlockChainImpl) WriteBlockWithState(
 	if err := rawdb.WriteCxLookupEntries(batch, block); err != nil {
 		return NonStatTy, err
 	}
-	if err := rawdb.WritePreimages(batch, block.NumberU64(), state.Preimages()); err != nil {
+	if err := rawdb.WritePreimages(bc.db, state.Preimages()); err != nil {
 		return NonStatTy, err
 	}
 
@@ -1677,7 +1679,7 @@ func (bc *BlockChainImpl) insertChain(chain types.Blocks, verifyHeaders bool) (i
 		} else {
 			parent = chain[i-1]
 		}
-		state, err := state.New(parent.Root(), bc.stateCache)
+		state, err := state.New(parent.Root(), bc.stateCache, nil)
 		if err != nil {
 			return i, events, coalescedLogs, err
 		}
@@ -3362,14 +3364,14 @@ func (bc *BlockChainImpl) tikvCleanCache() {
 		for i := bc.latestCleanCacheNum + 1; i <= to; i++ {
 			// build previous block statedb
 			fromBlock := bc.GetBlockByNumber(i)
-			fromTrie, err := state.New(fromBlock.Root(), bc.stateCache)
+			fromTrie, err := state.New(fromBlock.Root(), bc.stateCache, nil)
 			if err != nil {
 				continue
 			}
 
 			// build current block statedb
 			toBlock := bc.GetBlockByNumber(i + 1)
-			toTrie, err := state.New(toBlock.Root(), bc.stateCache)
+			toTrie, err := state.New(toBlock.Root(), bc.stateCache, nil)
 			if err != nil {
 				continue
 			}
@@ -3469,7 +3471,7 @@ func (bc *BlockChainImpl) InitTiKV(conf *harmonyconfig.TiKVConfig) {
 		// If redis is empty, the hit rate will be too low and the synchronization block speed will be slow
 		// set LOAD_PRE_FETCH is yes can significantly improve this.
 		if os.Getenv("LOAD_PRE_FETCH") == "yes" {
-			if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache); err == nil {
+			if trie, err := state.New(bc.CurrentBlock().Root(), bc.stateCache, nil); err == nil {
 				trie.Prefetch(512)
 			} else {
 				log.Println("LOAD_PRE_FETCH ERR: ", err)