Merge pull request #5849 from multiversx/migrate-data-trie-fix

Migrate data trie fix

cmd/node/config/enableEpochs.toml

@@ -258,7 +258,7 @@
     AutoBalanceDataTriesEnableEpoch = 1
 
     # MigrateDataTrieEnableEpoch represents the epoch when the data tries migration is enabled
-    MigrateDataTrieEnableEpoch = 999999
+    MigrateDataTrieEnableEpoch = 2
 
     # KeepExecOrderOnCreatedSCRsEnableEpoch represents the epoch when the execution order of created SCRs is ensured
     KeepExecOrderOnCreatedSCRsEnableEpoch = 1

go.mod

@@ -12,19 +12,20 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gops v0.3.18
 	github.com/gorilla/websocket v1.5.0
+	github.com/klauspost/cpuid/v2 v2.2.5
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/multiversx/mx-chain-communication-go v1.0.12
 	github.com/multiversx/mx-chain-core-go v1.2.18
 	github.com/multiversx/mx-chain-crypto-go v1.2.9
 	github.com/multiversx/mx-chain-es-indexer-go v1.4.18
 	github.com/multiversx/mx-chain-logger-go v1.0.13
-	github.com/multiversx/mx-chain-scenario-go v1.2.1
+	github.com/multiversx/mx-chain-scenario-go v1.3.0
 	github.com/multiversx/mx-chain-storage-go v1.0.14
-	github.com/multiversx/mx-chain-vm-common-go v1.5.10
-	github.com/multiversx/mx-chain-vm-go v1.5.24
-	github.com/multiversx/mx-chain-vm-v1_2-go v1.2.64
-	github.com/multiversx/mx-chain-vm-v1_3-go v1.3.65
-	github.com/multiversx/mx-chain-vm-v1_4-go v1.4.92
+	github.com/multiversx/mx-chain-vm-common-go v1.5.11
+	github.com/multiversx/mx-chain-vm-go v1.5.26
+	github.com/multiversx/mx-chain-vm-v1_2-go v1.2.65
+	github.com/multiversx/mx-chain-vm-v1_3-go v1.3.66
+	github.com/multiversx/mx-chain-vm-v1_4-go v1.4.94
 	github.com/pelletier/go-toml v1.9.3
 	github.com/pkg/errors v0.9.1
 	github.com/shirou/gopsutil v3.21.11+incompatible
@@ -91,7 +92,6 @@ require (
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.5 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/koron/go-ssdp v0.0.4 // indirect
 	github.com/leodido/go-urn v1.2.4 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect

go.sum

@@ -395,20 +395,20 @@ github.com/multiversx/mx-chain-es-indexer-go v1.4.18 h1:rCA+/mn/77MyB7c8FjtabdQe
 github.com/multiversx/mx-chain-es-indexer-go v1.4.18/go.mod h1:maraR9xXcfi0lLifhGMc+DVHpw1IOPX/c89HVckF1Js=
 github.com/multiversx/mx-chain-logger-go v1.0.13 h1:eru/TETo0MkO4ZTnXsQDKf4PBRpAXmqjT02klNT/JnY=
 github.com/multiversx/mx-chain-logger-go v1.0.13/go.mod h1:MZJhTAtZTJxT+yK2EHc4ZW3YOHUc1UdjCD0iahRNBZk=
-github.com/multiversx/mx-chain-scenario-go v1.2.1 h1:9eC6VcOEAKRRKZ7EbSWPLzCdNIMWwuNBtAZlgR4cSMA=
-github.com/multiversx/mx-chain-scenario-go v1.2.1/go.mod h1:EuZY7DpNFHVNSxJR8dKE1z2I8gBYfEFFPSwNUOXptqE=
+github.com/multiversx/mx-chain-scenario-go v1.3.0 h1:Vm6jYipJuIcJt8kamgXkePtKkwXaF5Lv5DglZjE5o8I=
+github.com/multiversx/mx-chain-scenario-go v1.3.0/go.mod h1:Sdgz8nzA9Ki/pdY/e2a4k90LALBKQB1Eo8HeCV3Bkzo=
 github.com/multiversx/mx-chain-storage-go v1.0.14 h1:h0acoqPS3FKJ4S3cKBEriTU0OabSQnpxai5WKhi1YCs=
 github.com/multiversx/mx-chain-storage-go v1.0.14/go.mod h1:sJ2q49tgjxNpMpsHysjABqCAB0FLBmDblbjBkQ8XfmA=
-github.com/multiversx/mx-chain-vm-common-go v1.5.10 h1:VoqVt9yX1nQUa0ZujMpdT3J3pKSnQcB6WCQLvIW4sqw=
-github.com/multiversx/mx-chain-vm-common-go v1.5.10/go.mod h1:sqkKMCnwkWl8DURdb9q7pctK8IANghdHY1KJLE0ox2c=
-github.com/multiversx/mx-chain-vm-go v1.5.24 h1:6RhMvf84Ys8DksDovms+su7w6j9TWz3Rtm/PpgV12Yw=
-github.com/multiversx/mx-chain-vm-go v1.5.24/go.mod h1:T03t+in5jqeTuFZKDt2wH/Sl9MSRczvWhmG+tQEIfec=
-github.com/multiversx/mx-chain-vm-v1_2-go v1.2.64 h1:3BEpSxEQibMMi4LXBjpo2y5vUa1LS7olDC2eDkmUfFQ=
-github.com/multiversx/mx-chain-vm-v1_2-go v1.2.64/go.mod h1:MUO2E4aEIu3siDkvjraO/WaBh/FxVeQyPWfsrZE+MTU=
-github.com/multiversx/mx-chain-vm-v1_3-go v1.3.65 h1:H0Duuoz6lR6KapqLqMspWTojaVtQRiLA5lIm6XV9H04=
-github.com/multiversx/mx-chain-vm-v1_3-go v1.3.65/go.mod h1:IZCHU3j/OSKVzdXu+5uZZSq2pVJrAS/KKAvnGrA/IKM=
-github.com/multiversx/mx-chain-vm-v1_4-go v1.4.92 h1:8ZcqnUQoIeM5k1F2IHvqbFzCumGwB4oVilWGuwurxpo=
-github.com/multiversx/mx-chain-vm-v1_4-go v1.4.92/go.mod h1:NyGULyeuEFe7Tb3gavT3Mti2oIFZJiMIf8VJIQnL4E8=
+github.com/multiversx/mx-chain-vm-common-go v1.5.11 h1:rAQR00ALKOmWAuNA8XW8hR02M9gmrAE4WZwyJH4dMMU=
+github.com/multiversx/mx-chain-vm-common-go v1.5.11/go.mod h1:T04rZt/VdwjPXcXVxltMmDmrhxNXsqlByYbWNWcNLYA=
+github.com/multiversx/mx-chain-vm-go v1.5.26 h1:ZjUJTG9cO2h5WNRIZ50ZSZNsTEPqXXPGS9Y/SAGyC2A=
+github.com/multiversx/mx-chain-vm-go v1.5.26/go.mod h1:gNZ/s4Z6OHg6ZeBsW6aDxWQduXsRS0Bsv4pfkmHeRzs=
+github.com/multiversx/mx-chain-vm-v1_2-go v1.2.65 h1:TxFjQH0dXC/ACQxlIgJbO7pVoh00rcqeKSnIjWTDMxg=
+github.com/multiversx/mx-chain-vm-v1_2-go v1.2.65/go.mod h1:UUUxIU7mlRkz+Jz4GWV2GkgJt2mKd+j1kky++RNYc9s=
+github.com/multiversx/mx-chain-vm-v1_3-go v1.3.66 h1:xgrXfHKa0Za4xjFj5W0FcYEivjrQIhLvGEFXXa1uQZU=
+github.com/multiversx/mx-chain-vm-v1_3-go v1.3.66/go.mod h1:mx6IOAqo7zjSinYd8D2YqlpMWsuqFoYXJ6bntnTOeQg=
+github.com/multiversx/mx-chain-vm-v1_4-go v1.4.94 h1:MZFEBjDmfwLGB0cZb/pvlLx+qRv/9tO83bEgHUk34is=
+github.com/multiversx/mx-chain-vm-v1_4-go v1.4.94/go.mod h1:uuSbZGe0UwOWQyHA4EeJWhs8UeDdhtmMwlhNaX9ppx0=
 github.com/multiversx/mx-components-big-int v1.0.0 h1:Wkr8lSzK2nDqixOrrBa47VNuqdhV1m/aJhaP1EMaiS8=
 github.com/multiversx/mx-components-big-int v1.0.0/go.mod h1:maIEMgHlNE2u78JaDD0oLzri+ShgU4okHfzP3LWGdQM=
 github.com/multiversx/protobuf v1.3.2 h1:RaNkxvGTGbA0lMcnHAN24qE1G1i+Xs5yHA6MDvQ4mSM=

integrationTests/multiShard/endOfEpoch/startInEpoch/startInEpoch_test.go

@@ -216,7 +216,7 @@ func testNodeStartsInEpoch(t *testing.T, shardID uint32, expectedHighestRound ui
 	cryptoComponents.BlKeyGen = &mock.KeyGenMock{}
 	cryptoComponents.TxKeyGen = &mock.KeyGenMock{}
 
-	coreComponents := integrationTests.GetDefaultCoreComponents()
+	coreComponents := integrationTests.GetDefaultCoreComponents(integrationTests.CreateEnableEpochsConfig())
 	coreComponents.InternalMarshalizerField = integrationTests.TestMarshalizer
 	coreComponents.TxMarshalizerField = integrationTests.TestMarshalizer
 	coreComponents.HasherField = integrationTests.TestHasher

integrationTests/multiShard/hardFork/hardFork_test.go

@@ -386,7 +386,7 @@ func hardForkImport(
 		defaults.FillGasMapInternal(gasSchedule, 1)
 		log.Warn("started import process")
 
-		coreComponents := integrationTests.GetDefaultCoreComponents()
+		coreComponents := integrationTests.GetDefaultCoreComponents(integrationTests.CreateEnableEpochsConfig())
 		coreComponents.InternalMarshalizerField = integrationTests.TestMarshalizer
 		coreComponents.TxMarshalizerField = integrationTests.TestMarshalizer
 		coreComponents.HasherField = integrationTests.TestHasher
@@ -558,7 +558,7 @@ func createHardForkExporter(
 		returnedConfigs[node.ShardCoordinator.SelfId()] = append(returnedConfigs[node.ShardCoordinator.SelfId()], exportConfig)
 		returnedConfigs[node.ShardCoordinator.SelfId()] = append(returnedConfigs[node.ShardCoordinator.SelfId()], keysConfig)
 
-		coreComponents := integrationTests.GetDefaultCoreComponents()
+		coreComponents := integrationTests.GetDefaultCoreComponents(integrationTests.CreateEnableEpochsConfig())
 		coreComponents.InternalMarshalizerField = integrationTests.TestMarshalizer
 		coreComponents.TxMarshalizerField = integrationTests.TestTxSignMarshalizer
 		coreComponents.HasherField = integrationTests.TestHasher

integrationTests/node/getAccount/getAccount_test.go

@@ -37,7 +37,7 @@ func TestNode_GetAccountAccountDoesNotExistsShouldRetEmpty(t *testing.T) {
 	accDB, _ := integrationTests.CreateAccountsDB(0, trieStorage)
 	rootHash, _ := accDB.Commit()
 
-	coreComponents := integrationTests.GetDefaultCoreComponents()
+	coreComponents := integrationTests.GetDefaultCoreComponents(integrationTests.CreateEnableEpochsConfig())
 	coreComponents.AddressPubKeyConverterField = integrationTests.TestAddressPubkeyConverter
 
 	dataComponents := integrationTests.GetDefaultDataComponents()
@@ -77,7 +77,7 @@ func TestNode_GetAccountAccountExistsShouldReturn(t *testing.T) {
 	testPubkey := integrationTests.CreateAccount(accDB, testNonce, testBalance)
 	rootHash, _ := accDB.Commit()
 
-	coreComponents := integrationTests.GetDefaultCoreComponents()
+	coreComponents := integrationTests.GetDefaultCoreComponents(integrationTests.CreateEnableEpochsConfig())
 	coreComponents.AddressPubKeyConverterField = testscommon.RealWorldBech32PubkeyConverter
 
 	dataComponents := integrationTests.GetDefaultDataComponents()

integrationTests/state/stateTrie/stateTrie_test.go

@@ -2,6 +2,7 @@ package stateTrie
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/hex"
@@ -24,11 +25,13 @@ import (
 	"github.com/multiversx/mx-chain-core-go/hashing/sha256"
 	crypto "github.com/multiversx/mx-chain-crypto-go"
 	"github.com/multiversx/mx-chain-go/common"
+	"github.com/multiversx/mx-chain-go/common/errChan"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
 	"github.com/multiversx/mx-chain-go/epochStart"
 	"github.com/multiversx/mx-chain-go/integrationTests"
 	"github.com/multiversx/mx-chain-go/integrationTests/mock"
+	esdtCommon "github.com/multiversx/mx-chain-go/integrationTests/vm/esdt"
 	"github.com/multiversx/mx-chain-go/sharding"
 	"github.com/multiversx/mx-chain-go/state"
 	"github.com/multiversx/mx-chain-go/state/factory"
@@ -2342,6 +2345,221 @@ func Test_SnapshotStateRemovesLastSnapshotStartedAfterSnapshotFinished(t *testin
 	assert.NotNil(t, err)
 }
 
+func TestMigrateDataTrieBuiltinFunc(t *testing.T) {
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
+
+	t.Run("migrate shard 0 system account", func(t *testing.T) {
+		shardId := byte(0)
+		nodes, idxProposers, nonce, round := startNodesAndIssueToken(t, 2, shardId)
+		defer func() {
+			for _, n := range nodes {
+				n.Close()
+			}
+		}()
+
+		valuesBeforeMigration := getValuesFromAccount(t, nodes[shardId].AccntState, core.SystemAccountAddress)
+		migrateDataTrieBuiltInFunc(t, nodes, shardId, core.SystemAccountAddress, nonce, round, idxProposers)
+		valuesAfterMigration := getValuesFromAccount(t, nodes[shardId].AccntState, core.SystemAccountAddress)
+
+		require.Equal(t, len(valuesBeforeMigration), len(valuesAfterMigration))
+		require.True(t, len(valuesAfterMigration) > 0)
+		for i := range valuesBeforeMigration {
+			require.Equal(t, valuesBeforeMigration[i], valuesAfterMigration[i])
+		}
+	})
+	t.Run("migrate shard 0 user account", func(t *testing.T) {
+		shardId := byte(0)
+		nodes, idxProposers, nonce, round := startNodesAndIssueToken(t, 2, shardId)
+		defer func() {
+			for _, n := range nodes {
+				n.Close()
+			}
+		}()
+
+		migrationAddress := nodes[shardId].OwnAccount.Address
+		valuesBeforeMigration := getValuesFromAccount(t, nodes[shardId].AccntState, migrationAddress)
+		migrateDataTrieBuiltInFunc(t, nodes, shardId, migrationAddress, nonce, round, idxProposers)
+		valuesAfterMigration := getValuesFromAccount(t, nodes[shardId].AccntState, migrationAddress)
+
+		require.Equal(t, len(valuesBeforeMigration), len(valuesAfterMigration))
+		require.True(t, len(valuesAfterMigration) > 0)
+		for i := range valuesBeforeMigration {
+			require.Equal(t, valuesBeforeMigration[i], valuesAfterMigration[i])
+		}
+	})
+	t.Run("migrate shard 1 system account", func(t *testing.T) {
+		shardId := byte(1)
+		nodes, idxProposers, nonce, round := startNodesAndIssueToken(t, 2, shardId)
+		defer func() {
+			for _, n := range nodes {
+				n.Close()
+			}
+		}()
+
+		valuesBeforeMigration := getValuesFromAccount(t, nodes[shardId].AccntState, core.SystemAccountAddress)
+		migrateDataTrieBuiltInFunc(t, nodes, shardId, core.SystemAccountAddress, nonce, round, idxProposers)
+		valuesAfterMigration := getValuesFromAccount(t, nodes[shardId].AccntState, core.SystemAccountAddress)
+
+		require.Equal(t, len(valuesBeforeMigration), len(valuesAfterMigration))
+		require.True(t, len(valuesAfterMigration) > 0)
+		for i := range valuesBeforeMigration {
+			require.Equal(t, valuesBeforeMigration[i], valuesAfterMigration[i])
+		}
+	})
+	t.Run("migrate shard 1 user account", func(t *testing.T) {
+		shardId := byte(1)
+		nodes, idxProposers, nonce, round := startNodesAndIssueToken(t, 2, shardId)
+		defer func() {
+			for _, n := range nodes {
+				n.Close()
+			}
+		}()
+
+		migrationAddress := nodes[shardId].OwnAccount.Address
+		valuesBeforeMigration := getValuesFromAccount(t, nodes[shardId].AccntState, migrationAddress)
+		migrateDataTrieBuiltInFunc(t, nodes, shardId, nodes[shardId].OwnAccount.Address, nonce, round, idxProposers)
+		valuesAfterMigration := getValuesFromAccount(t, nodes[shardId].AccntState, migrationAddress)
+
+		require.Equal(t, len(valuesBeforeMigration), len(valuesAfterMigration))
+		require.True(t, len(valuesAfterMigration) > 0)
+		for i := range valuesBeforeMigration {
+			require.Equal(t, valuesBeforeMigration[i], valuesAfterMigration[i])
+		}
+	})
+}
+
+func getValuesFromAccount(t *testing.T, adb state.AccountsAdapter, address []byte) [][]byte {
+	account, err := adb.GetExistingAccount(address)
+	require.Nil(t, err)
+
+	chLeaves := &common.TrieIteratorChannels{
+		LeavesChan: make(chan core.KeyValueHolder, common.TrieLeavesChannelDefaultCapacity),
+		ErrChan:    errChan.NewErrChanWrapper(),
+	}
+	err = account.(state.UserAccountHandler).GetAllLeaves(chLeaves, context.Background())
+	require.Nil(t, err)
+
+	values := make([][]byte, 0)
+	for leaf := range chLeaves.LeavesChan {
+		values = append(values, leaf.Value())
+	}
+
+	err = chLeaves.ErrChan.ReadFromChanNonBlocking()
+	require.Nil(t, err)
+
+	return values
+}
+
+func migrateDataTrieBuiltInFunc(
+	t *testing.T,
+	nodes []*integrationTests.TestProcessorNode,
+	shardId byte,
+	migrationAddress []byte,
+	nonce uint64,
+	round uint64,
+	idxProposers []int,
+) {
+	require.True(t, nodes[shardId].EnableEpochsHandler.IsAutoBalanceDataTriesEnabled())
+	isMigrated := getAddressMigrationStatus(t, nodes[shardId].AccntState, migrationAddress)
+	require.False(t, isMigrated)
+
+	integrationTests.CreateAndSendTransactionWithSenderAccount(nodes[shardId], nodes, big.NewInt(0), nodes[shardId].OwnAccount, getDestAccountAddress(migrationAddress, shardId), core.BuiltInFunctionMigrateDataTrie, 1000000)
+
+	time.Sleep(time.Second)
+	nrRoundsToPropagate := 5
+	_, _ = integrationTests.WaitOperationToBeDone(t, nodes, nrRoundsToPropagate, nonce, round, idxProposers)
+
+	isMigrated = getAddressMigrationStatus(t, nodes[shardId].AccntState, migrationAddress)
+	require.True(t, isMigrated)
+}
+
+func startNodesAndIssueToken(
+	t *testing.T,
+	numOfShards int,
+	issuerShardId byte,
+) ([]*integrationTests.TestProcessorNode, []int, uint64, uint64) {
+	nodesPerShard := 1
+	numMetachainNodes := 1
+
+	enableEpochs := config.EnableEpochs{
+		GlobalMintBurnDisableEpoch:                  integrationTests.UnreachableEpoch,
+		BuiltInFunctionOnMetaEnableEpoch:            integrationTests.UnreachableEpoch,
+		OptimizeGasUsedInCrossMiniBlocksEnableEpoch: integrationTests.UnreachableEpoch,
+		ScheduledMiniBlocksEnableEpoch:              integrationTests.UnreachableEpoch,
+		MiniBlockPartialExecutionEnableEpoch:        integrationTests.UnreachableEpoch,
+		StakingV2EnableEpoch:                        integrationTests.UnreachableEpoch,
+		AutoBalanceDataTriesEnableEpoch:             1,
+	}
+	nodes := integrationTests.CreateNodesWithEnableEpochs(
+		numOfShards,
+		nodesPerShard,
+		numMetachainNodes,
+		enableEpochs,
+	)
+
+	roundsPerEpoch := uint64(5)
+	for _, node := range nodes {
+		node.EpochStartTrigger.SetRoundsPerEpoch(roundsPerEpoch)
+	}
+
+	idxProposers := make([]int, numOfShards+1)
+	for i := 0; i < numOfShards; i++ {
+		idxProposers[i] = i * nodesPerShard
+	}
+	idxProposers[numOfShards] = numOfShards * nodesPerShard
+
+	integrationTests.DisplayAndStartNodes(nodes)
+
+	initialVal := int64(10000000000)
+	integrationTests.MintAllNodes(nodes, big.NewInt(initialVal))
+
+	round := uint64(0)
+	nonce := uint64(0)
+	round = integrationTests.IncrementAndPrintRound(round)
+	nonce++
+
+	// send token issue
+	initialSupply := int64(10000000000)
+	ticker := "TCK"
+	esdtCommon.IssueTestTokenWithIssuerAccount(nodes, nodes[issuerShardId].OwnAccount, initialSupply, ticker)
+
+	time.Sleep(time.Second)
+	nrRoundsToPropagate := 8
+	nonce, round = integrationTests.WaitOperationToBeDone(t, nodes, nrRoundsToPropagate, nonce, round, idxProposers)
+	time.Sleep(time.Second)
+
+	tokenIdentifier := string(integrationTests.GetTokenIdentifier(nodes, []byte(ticker)))
+
+	esdtCommon.CheckAddressHasTokens(t, nodes[issuerShardId].OwnAccount.Address, nodes, []byte(tokenIdentifier), 0, initialSupply)
+
+	return nodes, idxProposers, nonce, round
+}
+
+func getDestAccountAddress(migrationAddress []byte, shardId byte) []byte {
+	if bytes.Equal(migrationAddress, core.SystemAccountAddress) && shardId == 0 {
+		systemAccountAddress := bytes.Repeat([]byte{255}, 30)
+		systemAccountAddress = append(systemAccountAddress, []byte{0, 0}...)
+		return systemAccountAddress
+	}
+
+	return migrationAddress
+}
+
+func getAddressMigrationStatus(t *testing.T, adb state.AccountsAdapter, address []byte) bool {
+	account, err := adb.LoadAccount(address)
+	require.Nil(t, err)
+
+	userAccount, ok := account.(state.UserAccountHandler)
+	require.True(t, ok)
+
+	isMigrated, err := userAccount.DataTrie().IsMigratedToLatestVersion()
+	require.Nil(t, err)
+
+	return isMigrated
+}
+
 func addDataTriesForAccountsStartingWithIndex(
 	startIndex uint32,
 	nbAccounts uint32,

integrationTests/testConsensusNode.go

@@ -234,7 +234,7 @@ func (tcn *TestConsensusNode) initNode(args ArgsTestConsensusNode) {
 
 	tcn.initAccountsDB()
 
-	coreComponents := GetDefaultCoreComponents()
+	coreComponents := GetDefaultCoreComponents(CreateEnableEpochsConfig())
 	coreComponents.SyncTimerField = syncer
 	coreComponents.RoundHandlerField = roundHandler
 	coreComponents.InternalMarshalizerField = TestMarshalizer