New substate (#1066)

Apply new substate struct to Aida.

---------

Co-authored-by: wsodsong <wasuwee.sodsong@gmail.com>
Co-authored-by: Matěj Mlejnek <matej@fantom.foundation>
Co-authored-by: ph <ph@fantom.foundation>

cmd/aida-profile/profile/address_stats.go

@@ -18,7 +18,6 @@ package profile
 
 import (
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/urfave/cli/v2"
 )
@@ -30,7 +29,7 @@ var GetAddressStatsCommand = cli.Command{
 	Usage:     "computes usage statistics of addresses",
 	ArgsUsage: "<blockNumFirst> <blockNumLast>",
 	Flags: []cli.Flag{
-		&substate.WorkersFlag,
+		&utils.WorkersFlag,
 		&utils.AidaDbFlag,
 		&utils.ChainIDFlag,
 	},
@@ -50,11 +49,11 @@ Statistics on the usage of addresses are printed to the console.
 func getAddressStatsAction(ctx *cli.Context) error {
 	return getReferenceStatsAction(ctx, "address-stats", func(info *TransactionInfo) []common.Address {
 		addresses := []common.Address{}
-		for address := range info.st.InputAlloc {
-			addresses = append(addresses, address)
+		for address := range info.st.InputSubstate {
+			addresses = append(addresses, common.Address(address))
 		}
-		for address := range info.st.OutputAlloc {
-			addresses = append(addresses, address)
+		for address := range info.st.OutputSubstate {
+			addresses = append(addresses, common.Address(address))
 		}
 		return addresses
 	})

cmd/aida-profile/profile/codesize.go

@@ -20,7 +20,9 @@ import (
 	"fmt"
 
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
+	"github.com/Fantom-foundation/Substate/db"
+	"github.com/Fantom-foundation/Substate/substate"
+	substatetypes "github.com/Fantom-foundation/Substate/types"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/urfave/cli/v2"
 )
@@ -32,7 +34,7 @@ var GetCodeSizeCommand = cli.Command{
 	Usage:     "reports code size and nonce of smart contracts in the specified block range",
 	ArgsUsage: "<blockNumFirst> <blockNumLast>",
 	Flags: []cli.Flag{
-		&substate.WorkersFlag,
+		&utils.WorkersFlag,
 		&utils.AidaDbFlag,
 		&utils.ChainIDFlag,
 	},
@@ -46,11 +48,11 @@ last block of the inclusive range of blocks to replay transactions.
 Output log format: (block, timestamp, transaction, account, code size, nonce, transaction type)`,
 }
 
-func GetTxType(to *common.Address, alloc substate.SubstateAlloc) string {
+func GetTxType(to *common.Address, alloc substate.WorldState) string {
 	if to == nil {
 		return "create"
 	}
-	account, hasReceiver := alloc[*to]
+	account, hasReceiver := alloc[substatetypes.Address(*to)]
 	if to != nil && (!hasReceiver || len(account.Code) == 0) {
 		return "transfer"
 	}
@@ -61,27 +63,27 @@ func GetTxType(to *common.Address, alloc substate.SubstateAlloc) string {
 }
 
 // getCodeSizeTask returns codesize and nonce of accounts in a substate
-func getCodeSizeTask(block uint64, tx int, st *substate.Substate, taskPool *substate.SubstateTaskPool) error {
+func getCodeSizeTask(block uint64, tx int, st *substate.Substate, taskPool *db.SubstateTaskPool) error {
 	to := st.Message.To
 	timestamp := st.Env.Timestamp
-	txType := GetTxType(to, st.InputAlloc)
-	for account, accountInfo := range st.OutputAlloc {
+	txType := GetTxType((*common.Address)(to), st.InputSubstate)
+	for account, accountInfo := range st.OutputSubstate {
 		fmt.Printf("metric: %v,%v,%v,%v,%v,%v,%v\n",
 			block,
 			timestamp,
 			tx,
-			account.Hex(),
+			account.String(),
 			len(accountInfo.Code),
 			accountInfo.Nonce,
 			txType)
 	}
-	for account, accountInfo := range st.InputAlloc {
-		if _, found := st.OutputAlloc[account]; !found {
+	for account, accountInfo := range st.InputSubstate {
+		if _, found := st.OutputSubstate[account]; !found {
 			fmt.Printf("metric: %v,%v,%v,%v,%v,%v,%v\n",
 				block,
 				timestamp,
 				tx,
-				account.Hex(),
+				account.String(),
 				len(accountInfo.Code),
 				accountInfo.Nonce,
 				txType)
@@ -101,11 +103,12 @@ func getCodeSizeAction(ctx *cli.Context) error {
 
 	fmt.Printf("chain-id: %v\n", cfg.ChainID)
 
-	substate.SetSubstateDb(cfg.AidaDb)
-	substate.OpenSubstateDBReadOnly()
-	defer substate.CloseSubstateDB()
+	sdb, err := db.NewReadOnlySubstateDB(cfg.AidaDb)
+	if err != nil {
+		return fmt.Errorf("cannot open aida-db; %w", err)
+	}
 
-	taskPool := substate.NewSubstateTaskPool("aida-vm storage", getCodeSizeTask, cfg.First, cfg.Last, ctx)
+	taskPool := sdb.NewSubstateTaskPool("aida-vm storage", getCodeSizeTask, cfg.First, cfg.Last, ctx)
 	err = taskPool.Execute()
 	return err
 }

cmd/aida-profile/profile/key_stats.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/Fantom-foundation/Aida/logger"
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/urfave/cli/v2"
 )
@@ -33,7 +32,7 @@ var GetKeyStatsCommand = cli.Command{
 	Usage:     "computes usage statistics of accessed storage locations",
 	ArgsUsage: "<blockNumFirst> <blockNumLast>",
 	Flags: []cli.Flag{
-		&substate.WorkersFlag,
+		&utils.WorkersFlag,
 		&utils.AidaDbFlag,
 		&utils.ChainIDFlag,
 		&logger.LogLevelFlag,
@@ -54,14 +53,14 @@ Statistics on the usage of accessed storage locations are printed to the console
 func getKeyStatsAction(ctx *cli.Context) error {
 	return getReferenceStatsActionWithConsumer(ctx, "key-stats", func(info *TransactionInfo) []common.Hash {
 		keys := []common.Hash{}
-		for _, account := range info.st.InputAlloc {
+		for _, account := range info.st.InputSubstate {
 			for key := range account.Storage {
-				keys = append(keys, key)
+				keys = append(keys, common.Hash(key))
 			}
 		}
-		for _, account := range info.st.OutputAlloc {
+		for _, account := range info.st.InputSubstate {
 			for key := range account.Storage {
-				keys = append(keys, key)
+				keys = append(keys, common.Hash(key))
 			}
 		}
 		return keys

cmd/aida-profile/profile/location_stats.go

@@ -21,7 +21,6 @@ import (
 
 	"github.com/Fantom-foundation/Aida/logger"
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/urfave/cli/v2"
 )
@@ -33,7 +32,7 @@ var GetLocationStatsCommand = cli.Command{
 	Usage:     "computes usage statistics of accessed storage locations",
 	ArgsUsage: "<blockNumFirst> <blockNumLast>",
 	Flags: []cli.Flag{
-		&substate.WorkersFlag,
+		&utils.WorkersFlag,
 		&utils.AidaDbFlag,
 		&utils.ChainIDFlag,
 		&logger.LogLevelFlag,
@@ -82,17 +81,17 @@ func getLocationStatsAction(ctx *cli.Context) error {
 	var key_index Index[common.Hash]
 	return getReferenceStatsAction(ctx, "location-stats", func(info *TransactionInfo) []Location {
 		locations := []Location{}
-		for address, account := range info.st.InputAlloc {
-			address_id := address_index.Get(&address)
+		for address, account := range info.st.InputSubstate {
+			address_id := address_index.Get((*common.Address)(&address))
 			for key := range account.Storage {
-				key_id := key_index.Get(&key)
+				key_id := key_index.Get((*common.Hash)(&key))
 				locations = append(locations, Location{address_id, key_id})
 			}
 		}
-		for address, account := range info.st.OutputAlloc {
-			address_id := address_index.Get(&address)
+		for address, account := range info.st.OutputSubstate {
+			address_id := address_index.Get((*common.Address)(&address))
 			for key := range account.Storage {
-				key_id := key_index.Get(&key)
+				key_id := key_index.Get((*common.Hash)(&key))
 				locations = append(locations, Location{address_id, key_id})
 			}
 		}

cmd/aida-profile/profile/statistic.go

@@ -22,7 +22,8 @@ import (
 
 	"github.com/Fantom-foundation/Aida/logger"
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
+	"github.com/Fantom-foundation/Substate/db"
+	"github.com/Fantom-foundation/Substate/substate"
 	"github.com/urfave/cli/v2"
 )
 
@@ -90,7 +91,7 @@ func runStatCollector[T comparable](stats *AccessStatistics[T], src <-chan T, do
 }
 
 // collectAddressStats collects statistical information on address usage.
-func collectStats[T comparable](dest chan<- T, extract Extractor[T], block uint64, tx int, st *substate.Substate, taskPool *substate.SubstateTaskPool) error {
+func collectStats[T comparable](dest chan<- T, extract Extractor[T], block uint64, tx int, st *substate.Substate, taskPool *db.SubstateTaskPool) error {
 	info := TransactionInfo{
 		block: block,
 		tx:    tx,
@@ -131,9 +132,10 @@ func getReferenceStatsActionWithConsumer[T comparable](ctx *cli.Context, cli_com
 	// TODO this print has not been working ever since this functionality was introduced to aidaDb
 	//log.Infof("contract-db: %v\n", cfg.Db)
 
-	substate.SetSubstateDb(cfg.AidaDb)
-	substate.OpenSubstateDBReadOnly()
-	defer substate.CloseSubstateDB()
+	sdb, err := db.NewReadOnlySubstateDB(cfg.AidaDb)
+	if err != nil {
+		return fmt.Errorf("cannot open aida-db; %w", err)
+	}
 
 	// Start Collector.
 	stats := newStatistics[T](log)
@@ -142,12 +144,12 @@ func getReferenceStatsActionWithConsumer[T comparable](ctx *cli.Context, cli_com
 	go runStatCollector(&stats, refs, done)
 
 	// Create per-transaction task.
-	task := func(block uint64, tx int, st *substate.Substate, taskPool *substate.SubstateTaskPool) error {
+	task := func(block uint64, tx int, st *substate.Substate, taskPool *db.SubstateTaskPool) error {
 		return collectStats(refs, extract, block, tx, st, taskPool)
 	}
 
 	// Process all transactions in parallel, out-of-order.
-	taskPool := substate.NewSubstateTaskPool(fmt.Sprintf("aida-vm %v", cli_command), task, cfg.First, cfg.Last, ctx)
+	taskPool := sdb.NewSubstateTaskPool(fmt.Sprintf("aida-vm %v", cli_command), task, cfg.First, cfg.Last, ctx)
 	err = taskPool.Execute()
 	if err != nil {
 		return err

cmd/aida-profile/profile/storage.go

@@ -21,8 +21,9 @@ import (
 
 	"github.com/Fantom-foundation/Aida/logger"
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
-	"github.com/ethereum/go-ethereum/common"
+	"github.com/Fantom-foundation/Substate/db"
+	"github.com/Fantom-foundation/Substate/substate"
+	substatetypes "github.com/Fantom-foundation/Substate/types"
 	"github.com/urfave/cli/v2"
 )
 
@@ -33,7 +34,7 @@ var GetStorageUpdateSizeCommand = cli.Command{
 	Usage:     "returns changes in storage size by transactions in the specified block range",
 	ArgsUsage: "<blockNumFirst> <blockNumLast>",
 	Flags: []cli.Flag{
-		&substate.WorkersFlag,
+		&utils.WorkersFlag,
 		&utils.AidaDbFlag,
 		&utils.ChainIDFlag,
 		&logger.LogLevelFlag,
@@ -49,73 +50,73 @@ Output log format: (block, timestamp, transaction, account, storage update size,
 }
 
 // computeStorageSize computes the number of non-zero storage entries
-func computeStorageSizes(inUpdateSet map[common.Hash]common.Hash, outUpdateSet map[common.Hash]common.Hash) (int64, uint64, uint64) {
+func computeStorageSizes(inUpdateSet map[substatetypes.Hash]substatetypes.Hash, outUpdateSet map[substatetypes.Hash]substatetypes.Hash) (int64, uint64, uint64) {
 	deltaSize := int64(0)
 	inUpdateSize := uint64(0)
 	outUpdateSize := uint64(0)
 	wordSize := uint64(32) //bytes
 	for address, outValue := range outUpdateSet {
 		if inValue, found := inUpdateSet[address]; found {
-			if (inValue == common.Hash{} && outValue != common.Hash{}) {
+			if (inValue == substatetypes.Hash{} && outValue != substatetypes.Hash{}) {
 				// storage increases by one new cell
 				// (cell is empty in in-storage)
 				deltaSize++
-			} else if (inValue != common.Hash{} && outValue == common.Hash{}) {
+			} else if (inValue != substatetypes.Hash{} && outValue == substatetypes.Hash{}) {
 				// storage shrinks by one new cell
 				// (cell is empty in out-storage)
 				deltaSize--
 			}
 		} else {
 			// storage increases by one new cell
 			// (cell is not found in in-storage but found in out-storage)
-			if (outValue != common.Hash{}) {
+			if (outValue != substatetypes.Hash{}) {
 				deltaSize++
 			}
 		}
 		// compute update size
-		if (outValue != common.Hash{}) {
+		if (outValue != substatetypes.Hash{}) {
 			outUpdateSize++
 		}
 	}
 	for address, inValue := range inUpdateSet {
 		if _, found := outUpdateSet[address]; !found {
 			// storage shrinks by one cell
 			// (The cell does not exist for an address in in-storage)
-			if (inValue != common.Hash{}) {
+			if (inValue != substatetypes.Hash{}) {
 				deltaSize--
 			}
 		}
-		if (inValue != common.Hash{}) {
+		if (inValue != substatetypes.Hash{}) {
 			inUpdateSize++
 		}
 	}
 	return deltaSize * int64(wordSize), inUpdateSize * wordSize, outUpdateSize * wordSize
 }
 
 // getStorageUpdateSizeTask replays storage access of accounts in each transaction
-func getStorageUpdateSizeTask(block uint64, tx int, st *substate.Substate, taskPool *substate.SubstateTaskPool) error {
+func getStorageUpdateSizeTask(block uint64, tx int, st *substate.Substate, taskPool *db.SubstateTaskPool) error {
 
 	timestamp := st.Env.Timestamp
-	for wallet, outputAccount := range st.OutputAlloc {
+	for wallet, outputAccount := range st.OutputSubstate {
 		var (
 			deltaSize     int64
 			inUpdateSize  uint64
 			outUpdateSize uint64
 		)
 		// account exists in both input substate and output substate
-		if inputAccount, found := st.InputAlloc[wallet]; found {
+		if inputAccount, found := st.InputSubstate[wallet]; found {
 			deltaSize, inUpdateSize, outUpdateSize = computeStorageSizes(inputAccount.Storage, outputAccount.Storage)
 			// account exists in output substate but not input substate
 		} else {
-			deltaSize, inUpdateSize, outUpdateSize = computeStorageSizes(map[common.Hash]common.Hash{}, outputAccount.Storage)
+			deltaSize, inUpdateSize, outUpdateSize = computeStorageSizes(map[substatetypes.Hash]substatetypes.Hash{}, outputAccount.Storage)
 		}
-		fmt.Printf("metric: %v,%v,%v,%v,%v,%v,%v\n", block, timestamp, tx, wallet.Hex(), deltaSize, inUpdateSize, outUpdateSize)
+		fmt.Printf("metric: %v,%v,%v,%v,%v,%v,%v\n", block, timestamp, tx, wallet.String(), deltaSize, inUpdateSize, outUpdateSize)
 	}
 	// account exists in input substate but not output substate
-	for wallet, inputAccount := range st.InputAlloc {
-		if _, found := st.OutputAlloc[wallet]; !found {
-			deltaSize, inUpdateSize, outUpdateSize := computeStorageSizes(inputAccount.Storage, map[common.Hash]common.Hash{})
-			fmt.Printf("metric: %v,%v,%v,%v,%v,%v,%v\n", block, timestamp, tx, wallet.Hex(), deltaSize, inUpdateSize, outUpdateSize)
+	for wallet, inputAccount := range st.InputSubstate {
+		if _, found := st.OutputSubstate[wallet]; !found {
+			deltaSize, inUpdateSize, outUpdateSize := computeStorageSizes(inputAccount.Storage, map[substatetypes.Hash]substatetypes.Hash{})
+			fmt.Printf("metric: %v,%v,%v,%v,%v,%v,%v\n", block, timestamp, tx, wallet.String(), deltaSize, inUpdateSize, outUpdateSize)
 		}
 	}
 	return nil
@@ -134,11 +135,13 @@ func getStorageUpdateSizeAction(ctx *cli.Context) error {
 
 	log.Infof("chain-id: %v\n", cfg.ChainID)
 
-	substate.SetSubstateDb(cfg.AidaDb)
-	substate.OpenSubstateDBReadOnly()
-	defer substate.CloseSubstateDB()
+	sdb, err := db.NewReadOnlySubstateDB(cfg.AidaDb)
+	if err != nil {
+		return fmt.Errorf("cannot open aida-db; %w", err)
+	}
+	defer sdb.Close()
 
-	taskPool := substate.NewSubstateTaskPool("aida-vm storage", getStorageUpdateSizeTask, cfg.First, cfg.Last, ctx)
+	taskPool := sdb.NewSubstateTaskPool("aida-vm storage", getStorageUpdateSizeTask, cfg.First, cfg.Last, ctx)
 	err = taskPool.Execute()
 	return err
 }

cmd/aida-rpc/main.go

@@ -22,7 +22,6 @@ import (
 
 	"github.com/Fantom-foundation/Aida/logger"
 	"github.com/Fantom-foundation/Aida/utils"
-	substate "github.com/Fantom-foundation/Substate"
 	"github.com/urfave/cli/v2"
 )
 
@@ -35,7 +34,7 @@ func main() {
 		Copyright: "(c) 2023 Fantom Foundation",
 		Flags: []cli.Flag{
 			&utils.RpcRecordingFileFlag,
-			&substate.WorkersFlag,
+			&utils.WorkersFlag,
 
 			// VM
 			&utils.VmImplementation,

cmd/aida-rpc/run_rpc.go

@@ -109,5 +109,6 @@ func run(
 		},
 		processor,
 		extensionList,
+		nil,
 	)
 }