Nodes coordinator with staking v4

epochStart/bootstrap/baseStorageHandler.go

@@ -46,7 +46,7 @@ func (bsh *baseStorageHandler) groupMiniBlocksByShard(miniBlocks map[string]*blo
 
 func (bsh *baseStorageHandler) saveNodesCoordinatorRegistry(
 	metaBlock data.HeaderHandler,
-	nodesConfig *nodesCoordinator.NodesCoordinatorRegistry,
+	nodesConfig nodesCoordinator.NodesCoordinatorRegistryHandler,
 ) ([]byte, error) {
 	key := append([]byte(common.NodesCoordinatorRegistryKeyPrefix), metaBlock.GetPrevRandSeed()...)
 

epochStart/bootstrap/fromLocalStorage.go

@@ -2,7 +2,6 @@ package bootstrap
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"strconv"
 
@@ -192,19 +191,19 @@ func (e *epochStartBootstrap) prepareEpochFromStorage() (Parameters, error) {
 
 func (e *epochStartBootstrap) checkIfShuffledOut(
 	pubKey []byte,
-	nodesConfig *nodesCoordinator.NodesCoordinatorRegistry,
+	nodesConfig nodesCoordinator.NodesCoordinatorRegistryHandler,
 ) (uint32, bool) {
 	epochIDasString := fmt.Sprint(e.baseData.lastEpoch)
-	epochConfig := nodesConfig.EpochsConfig[epochIDasString]
+	epochConfig := nodesConfig.GetEpochsConfig()[epochIDasString]
 
-	newShardId, isWaitingForShard := checkIfPubkeyIsInMap(pubKey, epochConfig.WaitingValidators)
+	newShardId, isWaitingForShard := checkIfPubkeyIsInMap(pubKey, epochConfig.GetWaitingValidators())
 	if isWaitingForShard {
 		isShuffledOut := newShardId != e.baseData.shardId
 		e.nodeType = core.NodeTypeValidator
 		return newShardId, isShuffledOut
 	}
 
-	newShardId, isEligibleForShard := checkIfPubkeyIsInMap(pubKey, epochConfig.EligibleValidators)
+	newShardId, isEligibleForShard := checkIfPubkeyIsInMap(pubKey, epochConfig.GetEligibleValidators())
 	if isEligibleForShard {
 		isShuffledOut := newShardId != e.baseData.shardId
 		e.nodeType = core.NodeTypeValidator
@@ -245,7 +244,7 @@ func checkIfValidatorIsInList(
 	return false
 }
 
-func (e *epochStartBootstrap) getLastBootstrapData(storer storage.Storer) (*bootstrapStorage.BootstrapData, *nodesCoordinator.NodesCoordinatorRegistry, error) {
+func (e *epochStartBootstrap) getLastBootstrapData(storer storage.Storer) (*bootstrapStorage.BootstrapData, nodesCoordinator.NodesCoordinatorRegistryHandler, error) {
 	bootStorer, err := bootstrapStorage.NewBootstrapStorer(e.coreComponentsHolder.InternalMarshalizer(), storer)
 	if err != nil {
 		return nil, nil, err
@@ -264,8 +263,7 @@ func (e *epochStartBootstrap) getLastBootstrapData(storer storage.Storer) (*boot
 		return nil, nil, err
 	}
 
-	config := &nodesCoordinator.NodesCoordinatorRegistry{}
-	err = json.Unmarshal(d, config)
+	config, err := nodesCoordinator.CreateNodesCoordinatorRegistry(e.coreComponentsHolder.InternalMarshalizer(), d)
 	if err != nil {
 		return nil, nil, err
 	}

epochStart/bootstrap/interface.go

@@ -12,7 +12,7 @@ import (
 
 // StartOfEpochNodesConfigHandler defines the methods to process nodesConfig from epoch start metablocks
 type StartOfEpochNodesConfigHandler interface {
-	NodesConfigFromMetaBlock(currMetaBlock data.HeaderHandler, prevMetaBlock data.HeaderHandler) (*nodesCoordinator.NodesCoordinatorRegistry, uint32, error)
+	NodesConfigFromMetaBlock(currMetaBlock data.HeaderHandler, prevMetaBlock data.HeaderHandler) (nodesCoordinator.NodesCoordinatorRegistryHandler, uint32, error)
 	IsInterfaceNil() bool
 }
 
@@ -25,7 +25,7 @@ type EpochStartMetaBlockInterceptorProcessor interface {
 // StartInEpochNodesCoordinator defines the methods to process and save nodesCoordinator information to storage
 type StartInEpochNodesCoordinator interface {
 	EpochStartPrepare(metaHdr data.HeaderHandler, body data.BodyHandler)
-	NodesCoordinatorToRegistry() *nodesCoordinator.NodesCoordinatorRegistry
+	NodesCoordinatorToRegistry() nodesCoordinator.NodesCoordinatorRegistryHandler
 	ShardIdForEpoch(epoch uint32) (uint32, error)
 	IsInterfaceNil() bool
 }

epochStart/bootstrap/process.go

@@ -62,15 +62,15 @@ type Parameters struct {
 	Epoch       uint32
 	SelfShardId uint32
 	NumOfShards uint32
-	NodesConfig *nodesCoordinator.NodesCoordinatorRegistry
+	NodesConfig nodesCoordinator.NodesCoordinatorRegistryHandler
 }
 
 // ComponentsNeededForBootstrap holds the components which need to be initialized from network
 type ComponentsNeededForBootstrap struct {
 	EpochStartMetaBlock data.MetaHeaderHandler
 	PreviousEpochStart  data.MetaHeaderHandler
 	ShardHeader         data.HeaderHandler
-	NodesConfig         *nodesCoordinator.NodesCoordinatorRegistry
+	NodesConfig         nodesCoordinator.NodesCoordinatorRegistryHandler
 	Headers             map[string]data.HeaderHandler
 	ShardCoordinator    sharding.Coordinator
 	PendingMiniBlocks   map[string]*block.MiniBlock
@@ -126,7 +126,7 @@ type epochStartBootstrap struct {
 	epochStartMeta     data.MetaHeaderHandler
 	prevEpochStartMeta data.MetaHeaderHandler
 	syncedHeaders      map[string]data.HeaderHandler
-	nodesConfig        *nodesCoordinator.NodesCoordinatorRegistry
+	nodesConfig        nodesCoordinator.NodesCoordinatorRegistryHandler
 	baseData           baseDataInStorage
 	startRound         int64
 	nodeType           core.NodeType

epochStart/bootstrap/shardStorageHandler.go

@@ -104,7 +104,7 @@ func (ssh *shardStorageHandler) SaveDataToStorage(components *ComponentsNeededFo
 		return err
 	}
 
-	components.NodesConfig.CurrentEpoch = components.ShardHeader.GetEpoch()
+	components.NodesConfig.SetCurrentEpoch(components.ShardHeader.GetEpoch())
 	nodesCoordinatorConfigKey, err := ssh.saveNodesCoordinatorRegistry(components.EpochStartMetaBlock, components.NodesConfig)
 	if err != nil {
 		return err

epochStart/bootstrap/syncValidatorStatus.go

@@ -131,7 +131,7 @@ func NewSyncValidatorStatus(args ArgsNewSyncValidatorStatus) (*syncValidatorStat
 func (s *syncValidatorStatus) NodesConfigFromMetaBlock(
 	currMetaBlock data.HeaderHandler,
 	prevMetaBlock data.HeaderHandler,
-) (*nodesCoordinator.NodesCoordinatorRegistry, uint32, error) {
+) (nodesCoordinator.NodesCoordinatorRegistryHandler, uint32, error) {
 	if currMetaBlock.GetNonce() > 1 && !currMetaBlock.IsStartOfEpochBlock() {
 		return nil, 0, epochStart.ErrNotEpochStartBlock
 	}
@@ -155,7 +155,7 @@ func (s *syncValidatorStatus) NodesConfigFromMetaBlock(
 	}
 
 	nodesConfig := s.nodeCoordinator.NodesCoordinatorToRegistry()
-	nodesConfig.CurrentEpoch = currMetaBlock.GetEpoch()
+	nodesConfig.SetCurrentEpoch(currMetaBlock.GetEpoch())
 	return nodesConfig, selfShardId, nil
 }
 

factory/bootstrapParameters.go

@@ -25,7 +25,7 @@ func (bph *bootstrapParams) NumOfShards() uint32 {
 }
 
 // NodesConfig returns the nodes coordinator config after bootstrap
-func (bph *bootstrapParams) NodesConfig() *nodesCoordinator.NodesCoordinatorRegistry {
+func (bph *bootstrapParams) NodesConfig() nodesCoordinator.NodesCoordinatorRegistryHandler {
 	return bph.bootstrapParams.NodesConfig
 }
 

factory/interface.go

@@ -405,7 +405,7 @@ type BootstrapParamsHolder interface {
 	Epoch() uint32
 	SelfShardID() uint32
 	NumOfShards() uint32
-	NodesConfig() *nodesCoordinator.NodesCoordinatorRegistry
+	NodesConfig() nodesCoordinator.NodesCoordinatorRegistryHandler
 	IsInterfaceNil() bool
 }
 

factory/shardingFactory.go

@@ -142,15 +142,15 @@ func CreateNodesCoordinator(
 	if bootstrapParameters.NodesConfig() != nil {
 		nodeRegistry := bootstrapParameters.NodesConfig()
 		currentEpoch = bootstrapParameters.Epoch()
-		epochsConfig, ok := nodeRegistry.EpochsConfig[fmt.Sprintf("%d", currentEpoch)]
+		epochsConfig, ok := nodeRegistry.GetEpochsConfig()[fmt.Sprintf("%d", currentEpoch)]
 		if ok {
-			eligibles := epochsConfig.EligibleValidators
+			eligibles := epochsConfig.GetEligibleValidators()
 			eligibleValidators, err = nodesCoordinator.SerializableValidatorsToValidators(eligibles)
 			if err != nil {
 				return nil, err
 			}
 
-			waitings := epochsConfig.WaitingValidators
+			waitings := epochsConfig.GetWaitingValidators()
 			waitingValidators, err = nodesCoordinator.SerializableValidatorsToValidators(waitings)
 			if err != nil {
 				return nil, err

node/nodeRunner.go

@@ -799,7 +799,7 @@ func (nr *nodeRunner) logInformation(
 	log.Info("Bootstrap", "epoch", bootstrapComponents.EpochBootstrapParams().Epoch())
 	if bootstrapComponents.EpochBootstrapParams().NodesConfig() != nil {
 		log.Info("the epoch from nodesConfig is",
-			"epoch", bootstrapComponents.EpochBootstrapParams().NodesConfig().CurrentEpoch)
+			"epoch", bootstrapComponents.EpochBootstrapParams().NodesConfig().GetCurrentEpoch())
 	}
 
 	var shardIdString = core.GetShardIDString(bootstrapComponents.ShardCoordinator().SelfId())

sharding/nodesCoordinator/common.go

@@ -2,9 +2,11 @@ package nodesCoordinator
 
 import (
 	"encoding/hex"
+	"encoding/json"
 	"strconv"
 
 	"github.com/ElrondNetwork/elrond-go-core/core"
+	"github.com/ElrondNetwork/elrond-go-core/marshal"
 	logger "github.com/ElrondNetwork/elrond-go-logger"
 )
 
@@ -52,6 +54,7 @@ func displayNodesConfiguration(
 	waiting map[uint32][]Validator,
 	leaving map[uint32][]Validator,
 	actualRemaining map[uint32][]Validator,
+	shuffledOut map[uint32][]Validator,
 	nbShards uint32,
 ) {
 	for shard := uint32(0); shard <= nbShards; shard++ {
@@ -75,6 +78,10 @@ func displayNodesConfiguration(
 			pk := v.PubKey()
 			log.Debug("actually remaining", "pk", pk, "shardID", shardID)
 		}
+		for _, v := range shuffledOut[shardID] {
+			pk := v.PubKey()
+			log.Debug("shuffled out", "pk", pk, "shardID", shardID)
+		}
 	}
 }
 
@@ -108,3 +115,35 @@ func SerializableShardValidatorListToValidatorList(shardValidators []*Serializab
 	}
 	return newValidators, nil
 }
+
+// CreateNodesCoordinatorRegistry creates a NodesCoordinatorRegistryHandler depending on the buffer. Old version uses
+// NodesCoordinatorRegistry with a json marshaller; while the new version(from staking v4) uses NodesCoordinatorRegistryWithAuction
+// with proto marshaller
+func CreateNodesCoordinatorRegistry(marshaller marshal.Marshalizer, buff []byte) (NodesCoordinatorRegistryHandler, error) {
+	registry, err := createOldRegistry(buff)
+	if err == nil {
+		return registry, nil
+	}
+
+	return createRegistryWithAuction(marshaller, buff)
+}
+
+func createOldRegistry(buff []byte) (*NodesCoordinatorRegistry, error) {
+	registry := &NodesCoordinatorRegistry{}
+	err := json.Unmarshal(buff, registry)
+	if err != nil {
+		return nil, err
+	}
+
+	return registry, nil
+}
+
+func createRegistryWithAuction(marshaller marshal.Marshalizer, buff []byte) (*NodesCoordinatorRegistryWithAuction, error) {
+	registry := &NodesCoordinatorRegistryWithAuction{}
+	err := marshaller.Unmarshal(registry, buff)
+	if err != nil {
+		return nil, err
+	}
+
+	return registry, nil
+}

sharding/nodesCoordinator/dtos.go

@@ -7,6 +7,7 @@ type ArgsUpdateNodes struct {
 	NewNodes          []Validator
 	UnStakeLeaving    []Validator
 	AdditionalLeaving []Validator
+	Auction           []Validator
 	Rand              []byte
 	NbShards          uint32
 	Epoch             uint32
@@ -16,6 +17,7 @@ type ArgsUpdateNodes struct {
 type ResUpdateNodes struct {
 	Eligible       map[uint32][]Validator
 	Waiting        map[uint32][]Validator
+	ShuffledOut    map[uint32][]Validator
 	Leaving        []Validator
 	StillRemaining []Validator
 }

sharding/nodesCoordinator/hashValidatorShuffler.go

@@ -24,6 +24,7 @@ type NodesShufflerArgs struct {
 	MaxNodesEnableConfig           []config.MaxNodesChangeConfig
 	BalanceWaitingListsEnableEpoch uint32
 	WaitingListFixEnableEpoch      uint32
+	StakingV4EnableEpoch           uint32
 }
 
 type shuffleNodesArg struct {
@@ -32,6 +33,7 @@ type shuffleNodesArg struct {
 	unstakeLeaving          []Validator
 	additionalLeaving       []Validator
 	newNodes                []Validator
+	auction                 []Validator
 	randomness              []byte
 	distributor             ValidatorsDistributor
 	nodesMeta               uint32
@@ -40,6 +42,7 @@ type shuffleNodesArg struct {
 	maxNodesToSwapPerShard  uint32
 	flagBalanceWaitingLists bool
 	flagWaitingListFix      bool
+	flagStakingV4           bool
 }
 
 // TODO: Decide if transaction load statistics will be used for limiting the number of shards
@@ -61,6 +64,8 @@ type randHashShuffler struct {
 	flagBalanceWaitingLists        atomic.Flag
 	waitingListFixEnableEpoch      uint32
 	flagWaitingListFix             atomic.Flag
+	stakingV4EnableEpoch           uint32
+	flagStakingV4                  atomic.Flag
 }
 
 // NewHashValidatorsShuffler creates a validator shuffler that uses a hash between validator key and a given
@@ -85,10 +90,12 @@ func NewHashValidatorsShuffler(args *NodesShufflerArgs) (*randHashShuffler, erro
 		availableNodesConfigs:          configs,
 		balanceWaitingListsEnableEpoch: args.BalanceWaitingListsEnableEpoch,
 		waitingListFixEnableEpoch:      args.WaitingListFixEnableEpoch,
+		stakingV4EnableEpoch:           args.StakingV4EnableEpoch,
 	}
 
 	log.Debug("randHashShuffler: enable epoch for balance waiting list", "epoch", rxs.balanceWaitingListsEnableEpoch)
 	log.Debug("randHashShuffler: enable epoch for waiting waiting list", "epoch", rxs.waitingListFixEnableEpoch)
+	log.Debug("randHashShuffler: enable epoch for staking v4", "epoch", rxs.stakingV4EnableEpoch)
 
 	rxs.UpdateParams(args.NodesShard, args.NodesMeta, args.Hysteresis, args.Adaptivity)
 
@@ -176,6 +183,7 @@ func (rhs *randHashShuffler) UpdateNodeLists(args ArgsUpdateNodes) (*ResUpdateNo
 		unstakeLeaving:          args.UnStakeLeaving,
 		additionalLeaving:       args.AdditionalLeaving,
 		newNodes:                args.NewNodes,
+		auction:                 args.Auction,
 		randomness:              args.Rand,
 		nodesMeta:               nodesMeta,
 		nodesPerShard:           nodesPerShard,
@@ -184,6 +192,7 @@ func (rhs *randHashShuffler) UpdateNodeLists(args ArgsUpdateNodes) (*ResUpdateNo
 		maxNodesToSwapPerShard:  rhs.activeNodesConfig.NodesToShufflePerShard,
 		flagBalanceWaitingLists: rhs.flagBalanceWaitingLists.IsSet(),
 		flagWaitingListFix:      rhs.flagWaitingListFix.IsSet(),
+		flagStakingV4:           rhs.flagStakingV4.IsSet(),
 	})
 }
 
@@ -288,16 +297,24 @@ func shuffleNodes(arg shuffleNodesArg) (*ResUpdateNodes, error) {
 		log.Warn("distributeValidators newNodes failed", "error", err)
 	}
 
-	err = arg.distributor.DistributeValidators(newWaiting, shuffledOutMap, arg.randomness, arg.flagBalanceWaitingLists)
-	if err != nil {
-		log.Warn("distributeValidators shuffledOut failed", "error", err)
+	if arg.flagStakingV4 {
+		err = distributeValidators(newWaiting, arg.auction, arg.randomness, false)
+		if err != nil {
+			log.Warn("distributeValidators auction list failed", "error", err)
+		}
+	} else {
+		err = arg.distributor.DistributeValidators(newWaiting, shuffledOutMap, arg.randomness, arg.flagBalanceWaitingLists)
+		if err != nil {
+			log.Warn("distributeValidators shuffledOut failed", "error", err)
+		}
 	}
 
 	actualLeaving, _ := removeValidatorsFromList(allLeaving, stillRemainingInLeaving, len(stillRemainingInLeaving))
 
 	return &ResUpdateNodes{
 		Eligible:       newEligible,
 		Waiting:        newWaiting,
+		ShuffledOut:    shuffledOutMap,
 		Leaving:        actualLeaving,
 		StillRemaining: stillRemainingInLeaving,
 	}, nil
@@ -779,8 +796,12 @@ func (rhs *randHashShuffler) UpdateShufflerConfig(epoch uint32) {
 
 	rhs.flagBalanceWaitingLists.SetValue(epoch >= rhs.balanceWaitingListsEnableEpoch)
 	log.Debug("balanced waiting lists", "enabled", rhs.flagBalanceWaitingLists.IsSet())
+
 	rhs.flagWaitingListFix.SetValue(epoch >= rhs.waitingListFixEnableEpoch)
 	log.Debug("waiting list fix", "enabled", rhs.flagWaitingListFix.IsSet())
+
+	rhs.flagStakingV4.SetValue(epoch >= rhs.stakingV4EnableEpoch)
+	log.Debug("staking v4", "enabled", rhs.flagStakingV4.IsSet())
 }
 
 func (rhs *randHashShuffler) sortConfigs() {