POC: Distribute to waiting from auction based on leaving nodes

epochStart/dtos.go

@@ -15,3 +15,10 @@ type OwnerData struct {
 	AuctionList    []state.ValidatorInfoHandler
 	Qualified      bool
 }
+
+// ValidatorStatsInEpoch holds validator stats in an epoch
+type ValidatorStatsInEpoch struct {
+	Eligible map[uint32]int
+	Waiting  map[uint32]int
+	Leaving  map[uint32]int
+}

epochStart/interface.go

@@ -159,6 +159,7 @@ type StakingDataProvider interface {
 	ComputeUnQualifiedNodes(validatorInfos state.ShardValidatorsInfoMapHandler) ([][]byte, map[string][][]byte, error)
 	GetBlsKeyOwner(blsKey []byte) (string, error)
 	GetNumOfValidatorsInCurrentEpoch() uint32
+	GetCurrentEpochValidatorStats() ValidatorStatsInEpoch
 	GetOwnersData() map[string]*OwnerData
 	Clean()
 	IsInterfaceNil() bool

epochStart/metachain/auctionListSelector.go

@@ -198,7 +198,7 @@ func (als *auctionListSelector) SelectNodesFromAuctionList(
 
 	currNodesConfig := als.nodesConfigProvider.GetCurrentNodesConfig()
 	currNumOfValidators := als.stakingDataProvider.GetNumOfValidatorsInCurrentEpoch()
-	numOfShuffledNodes := currNodesConfig.NodesToShufflePerShard * (als.shardCoordinator.NumberOfShards() + 1)
+	numOfShuffledNodes, numForcedToStay := als.computeNumShuffledNodes(currNodesConfig)
 	numOfValidatorsAfterShuffling, err := safeSub(currNumOfValidators, numOfShuffledNodes)
 	if err != nil {
 		log.Warn(fmt.Sprintf("auctionListSelector.SelectNodesFromAuctionList: %v when trying to compute numOfValidatorsAfterShuffling = %v - %v (currNumOfValidators - numOfShuffledNodes)",
@@ -210,12 +210,12 @@ func (als *auctionListSelector) SelectNodesFromAuctionList(
 	}
 
 	maxNumNodes := currNodesConfig.MaxNumNodes
-	availableSlots, err := safeSub(maxNumNodes, numOfValidatorsAfterShuffling)
+	availableSlots, err := safeSub(maxNumNodes, numOfValidatorsAfterShuffling+numForcedToStay)
 	if availableSlots == 0 || err != nil {
-		log.Info(fmt.Sprintf("auctionListSelector.SelectNodesFromAuctionList: %v or zero value when trying to compute availableSlots = %v - %v (maxNodes - numOfValidatorsAfterShuffling); skip selecting nodes from auction list",
+		log.Info(fmt.Sprintf("auctionListSelector.SelectNodesFromAuctionList: %v or zero value when trying to compute availableSlots = %v - %v (maxNodes - numOfValidatorsAfterShuffling+numForcedToStay); skip selecting nodes from auction list",
 			err,
 			maxNumNodes,
-			numOfValidatorsAfterShuffling,
+			numOfValidatorsAfterShuffling+numForcedToStay,
 		))
 		return nil
 	}
@@ -224,9 +224,10 @@ func (als *auctionListSelector) SelectNodesFromAuctionList(
 		"max nodes", maxNumNodes,
 		"current number of validators", currNumOfValidators,
 		"num of nodes which will be shuffled out", numOfShuffledNodes,
+		"num forced to stay", numForcedToStay,
 		"num of validators after shuffling", numOfValidatorsAfterShuffling,
 		"auction list size", auctionListSize,
-		fmt.Sprintf("available slots (%v - %v)", maxNumNodes, numOfValidatorsAfterShuffling), availableSlots,
+		fmt.Sprintf("available slots (%v - %v)", maxNumNodes, numOfValidatorsAfterShuffling+numForcedToStay), availableSlots,
 	)
 
 	als.auctionListDisplayer.DisplayOwnersData(ownersData)
@@ -272,6 +273,53 @@ func isInAuction(validator state.ValidatorInfoHandler) bool {
 	return validator.GetList() == string(common.AuctionList)
 }
 
+func (als *auctionListSelector) computeNumShuffledNodes(currNodesConfig config.MaxNodesChangeConfig) (uint32, uint32) {
+	numNodesToShufflePerShard := currNodesConfig.NodesToShufflePerShard
+	numShuffledOut := numNodesToShufflePerShard * (als.shardCoordinator.NumberOfShards() + 1)
+	epochStats := als.stakingDataProvider.GetCurrentEpochValidatorStats()
+
+	actuallyNumLeaving := uint32(0)
+	forcedToStay := uint32(0)
+	for shardID := uint32(0); shardID < als.shardCoordinator.NumberOfShards(); shardID++ {
+		leavingInShard, forcedToStayInShard := computeActuallyNumLeaving(shardID, epochStats, numNodesToShufflePerShard)
+		actuallyNumLeaving += leavingInShard
+		forcedToStay += forcedToStayInShard
+	}
+
+	leavingInShard, forcedToStayInShard := computeActuallyNumLeaving(core.MetachainShardId, epochStats, numNodesToShufflePerShard)
+	actuallyNumLeaving += leavingInShard
+	forcedToStay += forcedToStayInShard
+
+	finalShuffledOut, err := safeSub(numShuffledOut, actuallyNumLeaving)
+	if err != nil {
+		log.Error("auctionListSelector.computeNumShuffledNodes", "error", err)
+		return numShuffledOut, 0
+	}
+
+	return finalShuffledOut, forcedToStay
+}
+
+func computeActuallyNumLeaving(shardID uint32, epochStats epochStart.ValidatorStatsInEpoch, numNodesToShuffledPerShard uint32) (uint32, uint32) {
+	numLeavingInShard := uint32(epochStats.Leaving[shardID])
+	numActiveInShard := uint32(epochStats.Waiting[shardID] + epochStats.Eligible[shardID])
+
+	log.Info("auctionListSelector.computeActuallyNumLeaving",
+		"shardID", shardID, "numLeavingInShard", numLeavingInShard, "numActiveInShard", numActiveInShard)
+
+	actuallyleaving := uint32(0)
+	forcedToStay := uint32(0)
+	if numLeavingInShard < numNodesToShuffledPerShard && numActiveInShard > numLeavingInShard {
+		actuallyleaving = numLeavingInShard
+	}
+
+	if numLeavingInShard > numNodesToShuffledPerShard {
+		actuallyleaving = numNodesToShuffledPerShard
+		forcedToStay = numLeavingInShard - numNodesToShuffledPerShard
+	}
+
+	return actuallyleaving, forcedToStay
+}
+
 // TODO: Move this in elrond-go-core
 func safeSub(a, b uint32) (uint32, error) {
 	if a < b {

epochStart/metachain/stakingDataProvider.go

@@ -46,6 +46,7 @@ type stakingDataProvider struct {
 	minNodePrice               *big.Int
 	numOfValidatorsInCurrEpoch uint32
 	enableEpochsHandler        common.EnableEpochsHandler
+	validatorStatsInEpoch      epochStart.ValidatorStatsInEpoch
 }
 
 // StakingDataProviderArgs is a struct placeholder for all arguments required to create a NewStakingDataProvider
@@ -77,6 +78,11 @@ func NewStakingDataProvider(args StakingDataProviderArgs) (*stakingDataProvider,
 		totalEligibleStake:      big.NewInt(0),
 		totalEligibleTopUpStake: big.NewInt(0),
 		enableEpochsHandler:     args.EnableEpochsHandler,
+		validatorStatsInEpoch: epochStart.ValidatorStatsInEpoch{
+			Eligible: make(map[uint32]int),
+			Waiting:  make(map[uint32]int),
+			Leaving:  make(map[uint32]int),
+		},
 	}
 
 	return sdp, nil
@@ -89,6 +95,11 @@ func (sdp *stakingDataProvider) Clean() {
 	sdp.totalEligibleStake.SetInt64(0)
 	sdp.totalEligibleTopUpStake.SetInt64(0)
 	sdp.numOfValidatorsInCurrEpoch = 0
+	sdp.validatorStatsInEpoch = epochStart.ValidatorStatsInEpoch{
+		Eligible: make(map[uint32]int),
+		Waiting:  make(map[uint32]int),
+		Leaving:  make(map[uint32]int),
+	}
 	sdp.mutStakingData.Unlock()
 }
 
@@ -200,6 +211,7 @@ func (sdp *stakingDataProvider) getAndFillOwnerStats(validator state.ValidatorIn
 		sdp.numOfValidatorsInCurrEpoch++
 	}
 
+	sdp.updateEpochStats(validator)
 	return ownerData, nil
 }
 
@@ -532,6 +544,48 @@ func (sdp *stakingDataProvider) GetNumOfValidatorsInCurrentEpoch() uint32 {
 	return sdp.numOfValidatorsInCurrEpoch
 }
 
+func (sdp *stakingDataProvider) updateEpochStats(validator state.ValidatorInfoHandler) {
+	validatorCurrentList := common.PeerType(validator.GetList())
+	shardID := validator.GetShardId()
+
+	if validatorCurrentList == common.EligibleList {
+		sdp.validatorStatsInEpoch.Eligible[shardID]++
+		return
+	}
+
+	if validatorCurrentList == common.WaitingList {
+		sdp.validatorStatsInEpoch.Waiting[shardID]++
+		return
+	}
+
+	validatorPreviousList := common.PeerType(validator.GetPreviousList())
+	if sdp.isValidatorLeaving(validatorCurrentList, validatorPreviousList) {
+		sdp.validatorStatsInEpoch.Leaving[shardID]++
+	}
+}
+
+func (sdp *stakingDataProvider) isValidatorLeaving(validatorCurrentList, validatorPreviousList common.PeerType) bool {
+	if validatorCurrentList != common.LeavingList {
+		return false
+	}
+
+	// If no previous list is set, means that staking v4 is not activated or node is leaving right before activation
+	// and this node will be considered as eligible by the nodes coordinator with legacy bug.
+	// Otherwise, it will have it set, and we should check its previous list in the current epoch
+	if len(validatorPreviousList) == 0 || validatorPreviousList == common.EligibleList || validatorPreviousList == common.WaitingList {
+		return true
+	}
+
+	return false
+}
+
+func (sdp *stakingDataProvider) GetCurrentEpochValidatorStats() epochStart.ValidatorStatsInEpoch {
+	sdp.mutStakingData.RLock()
+	defer sdp.mutStakingData.RUnlock()
+
+	return sdp.validatorStatsInEpoch
+}
+
 // IsInterfaceNil return true if underlying object is nil
 func (sdp *stakingDataProvider) IsInterfaceNil() bool {
 	return sdp == nil

epochStart/metachain/systemSCs_test.go

@@ -2093,7 +2093,7 @@ func TestSystemSCProcessor_ProcessSystemSmartContractStakingV4Enabled(t *testing
 	t.Parallel()
 
 	args, _ := createFullArgumentsForSystemSCProcessing(config.EnableEpochs{}, testscommon.CreateMemUnit())
-	nodesConfigProvider, _ := notifier.NewNodesConfigProvider(args.EpochNotifier, []config.MaxNodesChangeConfig{{MaxNumNodes: 8}})
+	nodesConfigProvider, _ := notifier.NewNodesConfigProvider(args.EpochNotifier, []config.MaxNodesChangeConfig{{MaxNumNodes: 9}})
 
 	auctionCfg := config.SoftAuctionConfig{
 		TopUpStep:             "10",
@@ -2179,7 +2179,7 @@ func TestSystemSCProcessor_ProcessSystemSmartContractStakingV4Enabled(t *testing
 		      will not participate in auction selection
 		    - owner6 does not have enough stake for 2 nodes => one of his auction nodes(pubKey14) will be unStaked at the end of the epoch =>
 		      his other auction node(pubKey15) will not participate in auction selection
-			- MaxNumNodes = 8
+			- MaxNumNodes = 9
 			- EligibleBlsKeys = 5 (pubKey0, pubKey1, pubKey3, pubKe13, pubKey17)
 			- QualifiedAuctionBlsKeys = 7 (pubKey2, pubKey4, pubKey5, pubKey7, pubKey9, pubKey10, pubKey11)
 			We can only select (MaxNumNodes - EligibleBlsKeys = 3) bls keys from AuctionList to be added to NewList

integrationTests/vm/staking/stakingV4_test.go

@@ -1524,3 +1524,166 @@ func TestStakingV4_LeavingNodesEdgeCases(t *testing.T) {
 	owner1LeftNodes := getIntersection(owner1NodesThatAreStillForcedToRemain, allCurrentNodesInSystem)
 	require.Zero(t, len(owner1LeftNodes))
 }
+
+// TODO if necessary:
+// - test with limit (unstake exactly 80 per shard)
+// - unstake more nodes when waiting lists are pretty empty
+
+func TestStakingV4LeavingNodesShouldDistributeToWaitingOnlyNecessaryNodes(t *testing.T) {
+	if testing.Short() {
+		t.Skip("this is not a short test")
+	}
+
+	numOfMetaNodes := uint32(400)
+	numOfShards := uint32(3)
+	numOfEligibleNodesPerShard := uint32(400)
+	numOfWaitingNodesPerShard := uint32(400)
+	numOfNodesToShufflePerShard := uint32(80)
+	shardConsensusGroupSize := 266
+	metaConsensusGroupSize := 266
+	numOfNodesInStakingQueue := uint32(80)
+
+	totalEligible := int(numOfEligibleNodesPerShard*numOfShards) + int(numOfMetaNodes) // 1600
+	totalWaiting := int(numOfWaitingNodesPerShard*numOfShards) + int(numOfMetaNodes)   // 1600
+
+	node := NewTestMetaProcessor(
+		numOfMetaNodes,
+		numOfShards,
+		numOfEligibleNodesPerShard,
+		numOfWaitingNodesPerShard,
+		numOfNodesToShufflePerShard,
+		shardConsensusGroupSize,
+		metaConsensusGroupSize,
+		numOfNodesInStakingQueue,
+	)
+	node.EpochStartTrigger.SetRoundsPerEpoch(4)
+
+	// 1. Check initial config is correct
+	initialNodes := node.NodesConfig
+	require.Len(t, getAllPubKeys(initialNodes.eligible), totalEligible)
+	require.Len(t, getAllPubKeys(initialNodes.waiting), totalWaiting)
+	require.Len(t, initialNodes.queue, int(numOfNodesInStakingQueue))
+	require.Empty(t, initialNodes.shuffledOut)
+	require.Empty(t, initialNodes.auction)
+
+	// 2. Check config after staking v4 initialization
+	node.Process(t, 5)
+	nodesConfigStakingV4Step1 := node.NodesConfig
+	require.Len(t, getAllPubKeys(nodesConfigStakingV4Step1.eligible), totalEligible)
+	require.Len(t, getAllPubKeys(nodesConfigStakingV4Step1.waiting), totalWaiting)
+	require.Empty(t, nodesConfigStakingV4Step1.queue)
+	require.Empty(t, nodesConfigStakingV4Step1.shuffledOut)
+	require.Empty(t, nodesConfigStakingV4Step1.auction) // the queue should be empty
+
+	// 3. re-stake the node nodes that were in the queue
+	node.ProcessReStake(t, initialNodes.queue)
+	nodesConfigStakingV4Step1 = node.NodesConfig
+	requireSameSliceDifferentOrder(t, initialNodes.queue, nodesConfigStakingV4Step1.auction)
+
+	node.Process(t, 10)
+
+	epochs := 0
+	prevConfig := node.NodesConfig
+	numOfSelectedNodesFromAuction := 320  // 320, since we will always fill shuffled out nodes with this config
+	numOfUnselectedNodesFromAuction := 80 // 80 = 400 from queue - 320
+	numOfShuffledOut := 80 * 4            // 80 per shard + meta
+	for epochs < 4 {
+		node.Process(t, 5)
+		newNodeConfig := node.NodesConfig
+
+		require.Len(t, getAllPubKeys(newNodeConfig.eligible), totalEligible) // 1600
+		require.Len(t, getAllPubKeys(newNodeConfig.waiting), 1280)           // 1280
+		require.Len(t, getAllPubKeys(newNodeConfig.shuffledOut), 320)        // 320
+		require.Len(t, newNodeConfig.auction, 400)                           // 400
+		require.Empty(t, newNodeConfig.queue)
+		require.Empty(t, newNodeConfig.leaving)
+
+		checkStakingV4EpochChangeFlow(t, newNodeConfig, prevConfig, numOfShuffledOut, numOfUnselectedNodesFromAuction, numOfSelectedNodesFromAuction)
+		prevConfig = newNodeConfig
+		epochs++
+	}
+
+	// UnStake:
+	// - 46 from waiting + eligible ( 13 waiting + 36 eligible)
+	// - 11 from auction
+	currNodesCfg := node.NodesConfig
+	nodesToUnstakeFromAuction := currNodesCfg.auction[:11]
+
+	nodesToUnstakeFromWaiting := append(currNodesCfg.waiting[0][:3], currNodesCfg.waiting[1][:3]...)
+	nodesToUnstakeFromWaiting = append(nodesToUnstakeFromWaiting, currNodesCfg.waiting[2][:3]...)
+	nodesToUnstakeFromWaiting = append(nodesToUnstakeFromWaiting, currNodesCfg.waiting[core.MetachainShardId][:4]...)
+
+	nodesToUnstakeFromEligible := append(currNodesCfg.eligible[0][:8], currNodesCfg.eligible[1][:8]...)
+	nodesToUnstakeFromEligible = append(nodesToUnstakeFromEligible, currNodesCfg.eligible[2][:8]...)
+	nodesToUnstakeFromEligible = append(nodesToUnstakeFromEligible, currNodesCfg.eligible[core.MetachainShardId][:9]...)
+
+	nodesToUnstake := getAllNodesToUnStake(nodesToUnstakeFromAuction, nodesToUnstakeFromWaiting, nodesToUnstakeFromEligible)
+
+	prevConfig = currNodesCfg
+	node.ProcessUnStake(t, nodesToUnstake)
+	node.Process(t, 5)
+	currNodesCfg = node.NodesConfig
+
+	require.Len(t, getAllPubKeys(currNodesCfg.leaving), 57)                                            // 11 auction + 46 active (13 waiting + 36 eligible)
+	require.Len(t, getAllPubKeys(currNodesCfg.shuffledOut), 274)                                       // 320 - 46 active
+	require.Len(t, currNodesCfg.auction, 343)                                                          // 400 initial - 57 leaving
+	requireSliceContainsNumOfElements(t, getAllPubKeys(currNodesCfg.waiting), prevConfig.auction, 320) // 320 selected
+	requireSliceContainsNumOfElements(t, currNodesCfg.auction, prevConfig.auction, 69)                 // 69 unselected
+
+	nodesToUnstakeFromAuction = make([][]byte, 0)
+	nodesToUnstakeFromWaiting = make([][]byte, 0)
+	nodesToUnstakeFromEligible = make([][]byte, 0)
+
+	prevConfig = currNodesCfg
+	// UnStake:
+	// - 224 from waiting + eligible ( 13 waiting + 36 eligible), but unbalanced:
+	//    -> unStake 100 from waiting shard=meta => will force to stay = 100 from meta
+	//	  -> unStake 90 from eligible shard=2 => will force to stay = 90 from shard 2
+	// - 11 from auction
+	nodesToUnstakeFromAuction = currNodesCfg.auction[:11]
+	nodesToUnstakeFromWaiting = append(currNodesCfg.waiting[0][:3], currNodesCfg.waiting[1][:3]...)
+	nodesToUnstakeFromWaiting = append(nodesToUnstakeFromWaiting, currNodesCfg.waiting[2][:3]...)
+	nodesToUnstakeFromWaiting = append(nodesToUnstakeFromWaiting, currNodesCfg.waiting[core.MetachainShardId][:100]...)
+
+	nodesToUnstakeFromEligible = append(currNodesCfg.eligible[0][:8], currNodesCfg.eligible[1][:8]...)
+	nodesToUnstakeFromEligible = append(nodesToUnstakeFromEligible, currNodesCfg.eligible[2][:90]...)
+	nodesToUnstakeFromEligible = append(nodesToUnstakeFromEligible, currNodesCfg.eligible[core.MetachainShardId][:9]...)
+
+	nodesToUnstake = getAllNodesToUnStake(nodesToUnstakeFromAuction, nodesToUnstakeFromWaiting, nodesToUnstakeFromEligible)
+	node.ProcessUnStake(t, nodesToUnstake)
+	node.Process(t, 5)
+	currNodesCfg = node.NodesConfig
+
+	// Leaving:
+	// - 11 auction
+	// - shard 0 = 11
+	// - shard 1 = 11
+	// - shard 2 = 80 (there were 93 unStakes, but only 80 will be leaving, rest 13 will be forced to stay)
+	// - shard meta = 80 (there were 109 unStakes, but only 80 will be leaving, rest 29 will be forced to stay)
+	// Therefore we will have in total actually leaving = 193 (11 + 11 + 11 + 80 + 80)
+	// We should see a log in selector like this:
+	// auctionListSelector.SelectNodesFromAuctionList max nodes = 2880 current number of validators = 2656 num of nodes which will be shuffled out = 138 num forced to stay = 42 num of validators after shuffling = 2518 auction list size = 332 available slots (2880 - 2560) = 320
+	require.Len(t, getAllPubKeys(currNodesCfg.leaving), 193)
+	require.Len(t, getAllPubKeys(currNodesCfg.shuffledOut), 138)                                       //    69 from shard0 + shard from shard1, rest will not be shuffled
+	require.Len(t, currNodesCfg.auction, 150)                                                          // 138 shuffled out + 12 unselected
+	requireSliceContainsNumOfElements(t, getAllPubKeys(currNodesCfg.waiting), prevConfig.auction, 320) // 320 selected
+	requireSliceContainsNumOfElements(t, currNodesCfg.auction, prevConfig.auction, 12)                 // 12 unselected
+}
+
+func getAllNodesToUnStake(nodesToUnStakeFromAuction, nodesToUnStakeFromWaiting, nodesToUnStakeFromEligible [][]byte) map[string][][]byte {
+	ret := make(map[string][][]byte)
+
+	for _, nodeToUnstake := range nodesToUnStakeFromAuction {
+		ret[string(nodeToUnstake)] = [][]byte{nodeToUnstake}
+	}
+
+	for _, nodeToUnstake := range nodesToUnStakeFromWaiting {
+		ret[string(nodeToUnstake)] = [][]byte{nodeToUnstake}
+	}
+
+	for _, nodeToUnstake := range nodesToUnStakeFromEligible {
+		ret[string(nodeToUnstake)] = [][]byte{nodeToUnstake}
+	}
+
+	return ret
+}

testscommon/stakingcommon/stakingDataProviderStub.go

@@ -88,6 +88,14 @@ func (sdps *StakingDataProviderStub) GetNumOfValidatorsInCurrentEpoch() uint32 {
 	return 0
 }
 
+func (sdps *StakingDataProviderStub) GetCurrentEpochValidatorStats() epochStart.ValidatorStatsInEpoch {
+	return epochStart.ValidatorStatsInEpoch{
+		Eligible: map[uint32]int{},
+		Waiting:  map[uint32]int{},
+		Leaving:  map[uint32]int{},
+	}
+}
+
 // GetOwnersData -
 func (sdps *StakingDataProviderStub) GetOwnersData() map[string]*epochStart.OwnerData {
 	if sdps.GetOwnersDataCalled != nil {