Merge pull request #6166 from multiversx/auction-from-shuffled-out

highlight nodes shuffled out to auction list

cmd/node/config/enableEpochs.toml

@@ -302,6 +302,10 @@
     # StakingV4Step3EnableEpoch represents the epoch in which selected nodes from auction will be distributed to waiting list
     StakingV4Step3EnableEpoch = 3
 
+    # CleanupAuctionOnLowWaitingListEnableEpoch represents the epoch when duplicated data cleanup from auction list is enabled in the condition of a low waiting list
+    # Should have the same value as StakingV4Step1EnableEpoch if the low waiting list has not happened, otherwise should have a greater value
+    CleanupAuctionOnLowWaitingListEnableEpoch = 1
+
     # AlwaysMergeContextsInEEIEnableEpoch represents the epoch in which the EEI will always merge the contexts
     AlwaysMergeContextsInEEIEnableEpoch = 1
 

common/constants.go

@@ -1011,6 +1011,7 @@ const (
 	StakingV4Step1Flag                                 core.EnableEpochFlag = "StakingV4Step1Flag"
 	StakingV4Step2Flag                                 core.EnableEpochFlag = "StakingV4Step2Flag"
 	StakingV4Step3Flag                                 core.EnableEpochFlag = "StakingV4Step3Flag"
+	CleanupAuctionOnLowWaitingListFlag                 core.EnableEpochFlag = "CleanupAuctionOnLowWaitingListFlag"
 	StakingV4StartedFlag                               core.EnableEpochFlag = "StakingV4StartedFlag"
 	AlwaysMergeContextsInEEIFlag                       core.EnableEpochFlag = "AlwaysMergeContextsInEEIFlag"
 	// all new flags must be added to createAllFlagsMap method, as part of enableEpochsHandler allFlagsDefined

common/enablers/enableEpochsHandler.go

@@ -6,10 +6,11 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	logger "github.com/multiversx/mx-chain-logger-go"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/process"
-	logger "github.com/multiversx/mx-chain-logger-go"
 )
 
 var log = logger.GetOrCreate("common/enablers")
@@ -713,6 +714,12 @@ func (handler *enableEpochsHandler) createAllFlagsMap() {
 			},
 			activationEpoch: handler.enableEpochsConfig.StakingV4Step3EnableEpoch,
 		},
+		common.CleanupAuctionOnLowWaitingListFlag: {
+			isActiveInEpoch: func(epoch uint32) bool {
+				return epoch >= handler.enableEpochsConfig.CleanupAuctionOnLowWaitingListEnableEpoch
+			},
+			activationEpoch: handler.enableEpochsConfig.CleanupAuctionOnLowWaitingListEnableEpoch,
+		},
 		common.StakingV4StartedFlag: {
 			isActiveInEpoch: func(epoch uint32) bool {
 				return epoch >= handler.enableEpochsConfig.StakingV4Step1EnableEpoch

common/enablers/enableEpochsHandler_test.go

@@ -5,13 +5,14 @@ import (
 	"testing"
 
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/testscommon/epochNotifier"
-	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func createEnableEpochsConfig() config.EnableEpochs {
@@ -113,6 +114,7 @@ func createEnableEpochsConfig() config.EnableEpochs {
 		StakingV4Step1EnableEpoch:                                96,
 		StakingV4Step2EnableEpoch:                                97,
 		StakingV4Step3EnableEpoch:                                98,
+		CleanupAuctionOnLowWaitingListEnableEpoch:                96,
 		AlwaysMergeContextsInEEIEnableEpoch:                      99,
 	}
 }
@@ -426,6 +428,7 @@ func TestEnableEpochsHandler_GetActivationEpoch(t *testing.T) {
 	require.Equal(t, cfg.StakingV4Step1EnableEpoch, handler.GetActivationEpoch(common.StakingV4Step1Flag))
 	require.Equal(t, cfg.StakingV4Step2EnableEpoch, handler.GetActivationEpoch(common.StakingV4Step2Flag))
 	require.Equal(t, cfg.StakingV4Step3EnableEpoch, handler.GetActivationEpoch(common.StakingV4Step3Flag))
+	require.Equal(t, cfg.CleanupAuctionOnLowWaitingListEnableEpoch, handler.GetActivationEpoch(common.CleanupAuctionOnLowWaitingListFlag))
 	require.Equal(t, cfg.StakingV4Step1EnableEpoch, handler.GetActivationEpoch(common.StakingV4StartedFlag))
 	require.Equal(t, cfg.AlwaysMergeContextsInEEIEnableEpoch, handler.GetActivationEpoch(common.AlwaysMergeContextsInEEIFlag))
 }

config/epochConfig.go

@@ -112,6 +112,7 @@ type EnableEpochs struct {
 	StakingV4Step1EnableEpoch                                uint32
 	StakingV4Step2EnableEpoch                                uint32
 	StakingV4Step3EnableEpoch                                uint32
+	CleanupAuctionOnLowWaitingListEnableEpoch                uint32
 	AlwaysMergeContextsInEEIEnableEpoch                      uint32
 	BLSMultiSignerEnableEpoch                                []MultiSignerConfig
 }

config/tomlConfig_test.go

@@ -5,10 +5,11 @@ import (
 	"strconv"
 	"testing"
 
-	p2pConfig "github.com/multiversx/mx-chain-go/p2p/config"
 	"github.com/pelletier/go-toml"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
+	p2pConfig "github.com/multiversx/mx-chain-go/p2p/config"
 )
 
 func TestTomlParser(t *testing.T) {
@@ -843,6 +844,9 @@ func TestEnableEpochConfig(t *testing.T) {
     # AlwaysMergeContextsInEEIEnableEpoch represents the epoch in which the EEI will always merge the contexts
     AlwaysMergeContextsInEEIEnableEpoch = 94
 
+    # CleanupAuctionOnLowWaitingListEnableEpoch represents the epoch when the cleanup auction on low waiting list is enabled
+    CleanupAuctionOnLowWaitingListEnableEpoch = 95
+
     # MaxNodesChangeEnableEpoch holds configuration for changing the maximum number of nodes and the enabling epoch
     MaxNodesChangeEnableEpoch = [
         { EpochEnable = 44, MaxNumNodes = 2169, NodesToShufflePerShard = 80 },
@@ -955,6 +959,7 @@ func TestEnableEpochConfig(t *testing.T) {
 			MigrateDataTrieEnableEpoch:                               92,
 			CurrentRandomnessOnSortingEnableEpoch:                    93,
 			AlwaysMergeContextsInEEIEnableEpoch:                      94,
+			CleanupAuctionOnLowWaitingListEnableEpoch:                95,
 			MaxNodesChangeEnableEpoch: []MaxNodesChangeConfig{
 				{
 					EpochEnable:            44,

epochStart/bootstrap/disabled/disabledNodesCoordinator.go

@@ -54,6 +54,11 @@ func (n *nodesCoordinator) GetAllShuffledOutValidatorsPublicKeys(_ uint32) (map[
 	return nil, nil
 }
 
+// GetShuffledOutToAuctionValidatorsPublicKeys -
+func (n *nodesCoordinator) GetShuffledOutToAuctionValidatorsPublicKeys(_ uint32) (map[uint32][][]byte, error) {
+	return nil, nil
+}
+
 // GetConsensusValidatorsPublicKeys -
 func (n *nodesCoordinator) GetConsensusValidatorsPublicKeys(_ []byte, _ uint64, _ uint32, _ uint32) ([]string, error) {
 	return nil, nil

integrationTests/chainSimulator/staking/stake/simpleStake_test.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
+	"github.com/stretchr/testify/require"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/integrationTests/chainSimulator/staking"
@@ -17,7 +19,6 @@ import (
 	"github.com/multiversx/mx-chain-go/node/chainSimulator/configs"
 	"github.com/multiversx/mx-chain-go/node/chainSimulator/process"
 	"github.com/multiversx/mx-chain-go/vm"
-	"github.com/stretchr/testify/require"
 )
 
 // Test scenarios
@@ -261,30 +262,30 @@ func TestChainSimulator_StakingV4Step2APICalls(t *testing.T) {
 		err = cs.GenerateBlocks(2)
 		require.Nil(t, err)
 
-		numQualified, numUnQualified := getNumQualifiedAndUnqualified(t, metachainNode)
-		require.Equal(t, 8, numQualified)
-		require.Equal(t, 1, numUnQualified)
+		qualified, unQualified := getQualifiedAndUnqualifiedNodes(t, metachainNode)
+		require.Equal(t, 8, len(qualified))
+		require.Equal(t, 1, len(unQualified))
 	}
 }
 
-func getNumQualifiedAndUnqualified(t *testing.T, metachainNode process.NodeHandler) (int, int) {
+func getQualifiedAndUnqualifiedNodes(t *testing.T, metachainNode process.NodeHandler) ([]string, []string) {
 	err := metachainNode.GetProcessComponents().ValidatorsProvider().ForceUpdate()
 	require.Nil(t, err)
 	auctionList, err := metachainNode.GetProcessComponents().ValidatorsProvider().GetAuctionList()
 	require.Nil(t, err)
 
-	numQualified := 0
-	numUnQualified := 0
+	qualified := make([]string, 0)
+	unQualified := make([]string, 0)
 
 	for _, auctionOwnerData := range auctionList {
 		for _, auctionNode := range auctionOwnerData.Nodes {
 			if auctionNode.Qualified {
-				numQualified++
+				qualified = append(qualified, auctionNode.BlsKey)
 			} else {
-				numUnQualified++
+				unQualified = append(unQualified, auctionNode.BlsKey)
 			}
 		}
 	}
 
-	return numQualified, numUnQualified
+	return qualified, unQualified
 }

integrationTests/chainSimulator/staking/stake/stakeAndUnStake_test.go

@@ -12,6 +12,9 @@ import (
 	coreAPI "github.com/multiversx/mx-chain-core-go/data/api"
 	"github.com/multiversx/mx-chain-core-go/data/transaction"
 	"github.com/multiversx/mx-chain-core-go/data/validator"
+	logger "github.com/multiversx/mx-chain-logger-go"
+	"github.com/stretchr/testify/require"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	chainSimulatorIntegrationTests "github.com/multiversx/mx-chain-go/integrationTests/chainSimulator"
@@ -23,8 +26,6 @@ import (
 	chainSimulatorProcess "github.com/multiversx/mx-chain-go/node/chainSimulator/process"
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/vm"
-	logger "github.com/multiversx/mx-chain-logger-go"
-	"github.com/stretchr/testify/require"
 )
 
 const (
@@ -2347,6 +2348,7 @@ func TestChainSimulator_UnStakeOneActiveNodeAndCheckAPIAuctionList(t *testing.T)
 			cfg.EpochConfig.EnableEpochs.StakingV4Step1EnableEpoch = stakingV4Step1Epoch
 			cfg.EpochConfig.EnableEpochs.StakingV4Step2EnableEpoch = stakingV4Step2Epoch
 			cfg.EpochConfig.EnableEpochs.StakingV4Step3EnableEpoch = stakingV4Step3Epoch
+			cfg.EpochConfig.EnableEpochs.CleanupAuctionOnLowWaitingListEnableEpoch = stakingV4Step1Epoch
 
 			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[1].MaxNumNodes = 32
 			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[1].NodesToShufflePerShard = 2
@@ -2366,24 +2368,153 @@ func TestChainSimulator_UnStakeOneActiveNodeAndCheckAPIAuctionList(t *testing.T)
 
 	metachainNode := cs.GetNodeHandler(core.MetachainShardId)
 
-	numQualified, numUnQualified := getNumQualifiedAndUnqualified(t, metachainNode)
-	require.Equal(t, 8, numQualified)
-	require.Equal(t, 0, numUnQualified)
+	qualified, unQualified := getQualifiedAndUnqualifiedNodes(t, metachainNode)
+	require.Equal(t, 8, len(qualified))
+	require.Equal(t, 0, len(unQualified))
 
 	stakeOneNode(t, cs)
 
-	numQualified, numUnQualified = getNumQualifiedAndUnqualified(t, metachainNode)
-	require.Equal(t, 8, numQualified)
-	require.Equal(t, 1, numUnQualified)
+	qualified, unQualified = getQualifiedAndUnqualifiedNodes(t, metachainNode)
+	require.Equal(t, 8, len(qualified))
+	require.Equal(t, 1, len(unQualified))
 
 	unStakeOneActiveNode(t, cs)
 
-	numQualified, numUnQualified = getNumQualifiedAndUnqualified(t, metachainNode)
-	require.Equal(t, 8, numQualified)
-	require.Equal(t, 1, numUnQualified)
+	qualified, unQualified = getQualifiedAndUnqualifiedNodes(t, metachainNode)
+	require.Equal(t, 8, len(qualified))
+	require.Equal(t, 1, len(unQualified))
+}
+
+// Nodes configuration at genesis consisting of a total of 40 nodes, distributed on 3 shards + meta:
+// - 4 eligible nodes/shard
+// - 4 waiting nodes/shard
+// - 2 nodes to shuffle per shard
+// - max num nodes config for stakingV4 step3 = 32 (being downsized from previously 40 nodes)
+// - with this config, we should always select max 8 nodes from auction list if there are > 40 nodes in the network
+// This test will run with only 32 nodes and check that there are no nodes in the auction list,
+// because of the lowWaitingList condition being triggered when in staking v4
+func TestChainSimulator_EdgeCaseLowWaitingList(t *testing.T) {
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
+
+	startTime := time.Now().Unix()
+	roundDurationInMillis := uint64(6000)
+	roundsPerEpoch := core.OptionalUint64{
+		HasValue: true,
+		Value:    20,
+	}
+
+	stakingV4Step1Epoch := uint32(2)
+	stakingV4Step2Epoch := uint32(3)
+	stakingV4Step3Epoch := uint32(4)
+
+	numOfShards := uint32(3)
+	cs, err := chainSimulator.NewChainSimulator(chainSimulator.ArgsChainSimulator{
+		BypassTxSignatureCheck:   false,
+		TempDir:                  t.TempDir(),
+		PathToInitialConfig:      defaultPathToInitialConfig,
+		NumOfShards:              numOfShards,
+		GenesisTimestamp:         startTime,
+		RoundDurationInMillis:    roundDurationInMillis,
+		RoundsPerEpoch:           roundsPerEpoch,
+		ApiInterface:             api.NewNoApiInterface(),
+		MinNodesPerShard:         4,
+		MetaChainMinNodes:        4,
+		NumNodesWaitingListMeta:  2,
+		NumNodesWaitingListShard: 2,
+		AlterConfigsFunction: func(cfg *config.Configs) {
+			cfg.EpochConfig.EnableEpochs.StakingV4Step1EnableEpoch = stakingV4Step1Epoch
+			cfg.EpochConfig.EnableEpochs.StakingV4Step2EnableEpoch = stakingV4Step2Epoch
+			cfg.EpochConfig.EnableEpochs.StakingV4Step3EnableEpoch = stakingV4Step3Epoch
+
+			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[1].MaxNumNodes = 40
+			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[1].NodesToShufflePerShard = 2
+
+			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[2].EpochEnable = stakingV4Step3Epoch
+			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[2].MaxNumNodes = 32
+			cfg.EpochConfig.EnableEpochs.MaxNodesChangeEnableEpoch[2].NodesToShufflePerShard = 2
+		},
+	})
+	require.Nil(t, err)
+	require.NotNil(t, cs)
+
+	defer cs.Close()
+
+	epochToCheck := int32(stakingV4Step3Epoch + 1)
+	err = cs.GenerateBlocksUntilEpochIsReached(epochToCheck)
+	require.Nil(t, err)
+
+	metachainNode := cs.GetNodeHandler(core.MetachainShardId)
+	qualified, unQualified := getQualifiedAndUnqualifiedNodes(t, metachainNode)
+	require.Equal(t, 0, len(qualified))
+	require.Equal(t, 0, len(unQualified))
+
+	// we always have 0 in auction list because of the lowWaitingList condition
+	epochToCheck += 1
+	err = cs.GenerateBlocksUntilEpochIsReached(epochToCheck)
+	require.Nil(t, err)
+
+	qualified, unQualified = getQualifiedAndUnqualifiedNodes(t, metachainNode)
+	require.Equal(t, 0, len(qualified))
+	require.Equal(t, 0, len(unQualified))
+
+	// stake 16 mode nodes, these will go to auction list
+	stakeNodes(t, cs, 17)
+
+	epochToCheck += 1
+	err = cs.GenerateBlocksUntilEpochIsReached(epochToCheck)
+	require.Nil(t, err)
+
+	qualified, unQualified = getQualifiedAndUnqualifiedNodes(t, metachainNode)
+	// all the previously registered will be selected, as we have 24 nodes in eligible+waiting, 8 will shuffle out,
+	// but this time there will be not be lowWaitingList, as there are enough in auction, so we will end up with
+	// 24-8 = 16 nodes remaining + 16 from auction, to fill up all 32 positions
+	require.Equal(t, 16, len(qualified))
+	require.Equal(t, 1, len(unQualified))
+
+	shuffledOutNodesKeys, err := metachainNode.GetProcessComponents().NodesCoordinator().GetShuffledOutToAuctionValidatorsPublicKeys(uint32(epochToCheck))
+	require.Nil(t, err)
+
+	checkKeysNotInMap(t, shuffledOutNodesKeys, qualified)
+	checkKeysNotInMap(t, shuffledOutNodesKeys, unQualified)
+}
+
+func checkKeysNotInMap(t *testing.T, m map[uint32][][]byte, keys []string) {
+	for _, key := range keys {
+		for _, v := range m {
+			for _, k := range v {
+				mapKey := hex.EncodeToString(k)
+				require.NotEqual(t, key, mapKey)
+			}
+		}
+	}
+}
+
+func stakeNodes(t *testing.T, cs chainSimulatorIntegrationTests.ChainSimulator, numNodesToStake int) {
+	txs := make([]*transaction.Transaction, numNodesToStake)
+	for i := 0; i < numNodesToStake; i++ {
+		txs[i] = createStakeTransaction(t, cs)
+	}
+
+	stakeTxs, err := cs.SendTxsAndGenerateBlocksTilAreExecuted(txs, staking.MaxNumOfBlockToGenerateWhenExecutingTx)
+	require.Nil(t, err)
+	require.NotNil(t, stakeTxs)
+	require.Len(t, stakeTxs, numNodesToStake)
+
+	require.Nil(t, cs.GenerateBlocks(1))
 }
 
 func stakeOneNode(t *testing.T, cs chainSimulatorIntegrationTests.ChainSimulator) {
+	txStake := createStakeTransaction(t, cs)
+	stakeTx, err := cs.SendTxAndGenerateBlockTilTxIsExecuted(txStake, staking.MaxNumOfBlockToGenerateWhenExecutingTx)
+	require.Nil(t, err)
+	require.NotNil(t, stakeTx)
+
+	require.Nil(t, cs.GenerateBlocks(1))
+}
+
+func createStakeTransaction(t *testing.T, cs chainSimulatorIntegrationTests.ChainSimulator) *transaction.Transaction {
 	privateKey, blsKeys, err := chainSimulator.GenerateBlsPrivateKeys(1)
 	require.Nil(t, err)
 	err = cs.AddValidatorKeys(privateKey)
@@ -2394,12 +2525,7 @@ func stakeOneNode(t *testing.T, cs chainSimulatorIntegrationTests.ChainSimulator
 	require.Nil(t, err)
 
 	txDataField := fmt.Sprintf("stake@01@%s@%s", blsKeys[0], staking.MockBLSSignature)
-	txStake := staking.GenerateTransaction(validatorOwner.Bytes, 0, vm.ValidatorSCAddress, staking.MinimumStakeValue, txDataField, staking.GasLimitForStakeOperation)
-	stakeTx, err := cs.SendTxAndGenerateBlockTilTxIsExecuted(txStake, staking.MaxNumOfBlockToGenerateWhenExecutingTx)
-	require.Nil(t, err)
-	require.NotNil(t, stakeTx)
-
-	require.Nil(t, cs.GenerateBlocks(1))
+	return staking.GenerateTransaction(validatorOwner.Bytes, 0, vm.ValidatorSCAddress, staking.MinimumStakeValue, txDataField, staking.GasLimitForStakeOperation)
 }
 
 func unStakeOneActiveNode(t *testing.T, cs chainSimulatorIntegrationTests.ChainSimulator) {