diff --git a/local/blockchain.go b/local/blockchain.go index a81f7d72..bd46a729 100644 --- a/local/blockchain.go +++ b/local/blockchain.go @@ -159,9 +159,19 @@ func (ln *localNetwork) RegisterBlockchainAliases( return nil } +func (ln *localNetwork) AddSubnetValidators( + ctx context.Context, + specs []network.SubnetValidatorSpec, +) error { + ln.lock.Lock() + defer ln.lock.Unlock() + + return ln.addSubnetValidators(ctx, specs) +} + func (ln *localNetwork) RemoveSubnetValidators( ctx context.Context, - removeSubnetSpecs []network.RemoveSubnetValidatorSpec, + removeSubnetSpecs []network.SubnetValidatorSpec, ) error { ln.lock.Lock() defer ln.lock.Unlock() @@ -305,7 +315,7 @@ func (ln *localNetwork) installCustomChains( return nil, err } - if err = ln.addSubnetValidators(ctx, platformCli, w, subnetIDs, subnetSpecs); err != nil { + if err = ln.sendAddSubnetValidatorTxs(ctx, platformCli, w, subnetIDs, subnetSpecs); err != nil { return nil, err } @@ -369,6 +379,86 @@ func (ln *localNetwork) installCustomChains( return chainInfos, nil } +func (ln *localNetwork) addSubnetValidators( + ctx context.Context, + subnetValidatorsSpecs []network.SubnetValidatorSpec, +) error { + ln.log.Info(logging.Blue.Wrap(logging.Bold.Wrap("add subnet validators"))) + + clientURI, err := ln.getClientURI() + if err != nil { + return err + } + platformCli := platformvm.NewClient(clientURI) + + // wallet needs txs for all previously created subnets + subnetIDs := make([]ids.ID, len(subnetValidatorsSpecs)) + for i, spec := range subnetValidatorsSpecs { + subnetID, err := ids.FromString(spec.SubnetID) + if err != nil { + return err + } + subnetIDs[i] = subnetID + } + wallet, err := newWallet(ctx, clientURI, subnetIDs) + if err != nil { + return err + } + + for _, spec := range subnetValidatorsSpecs { + if len(spec.NodeNames) == 0 { + return fmt.Errorf("no validators provided for subnet %s", spec.SubnetID) + } + } + + // create new nodes + for _, spec := range subnetValidatorsSpecs { + for _, nodeName := range spec.NodeNames { + _, ok := ln.nodes[nodeName] + if !ok { + ln.log.Info(logging.Green.Wrap(fmt.Sprintf("adding new participant %s", nodeName))) + if _, err := ln.addNode(node.Config{ + Name: nodeName, + }); err != nil { + return err + } + } + } + } + if err := ln.healthy(ctx); err != nil { + return err + } + + // just ensure all nodes are primary validators (so can be subnet validators) + if err := ln.addPrimaryValidators(ctx, platformCli, wallet); err != nil { + return err + } + + // wait for nodes to be primary validators before trying to add them as subnet ones + if err = ln.waitPrimaryValidators(ctx, platformCli); err != nil { + return err + } + + subnetSpecs := []network.SubnetSpec{} + for _, spec := range subnetValidatorsSpecs { + subnetSpecs = append(subnetSpecs, network.SubnetSpec{Participants: spec.NodeNames}) + } + + if err = ln.sendAddSubnetValidatorTxs(ctx, platformCli, wallet, subnetIDs, subnetSpecs); err != nil { + return err + } + + if err := ln.restartNodes(ctx, subnetIDs, subnetSpecs, nil, nil, nil); err != nil { + return err + } + + if err = ln.waitSubnetValidators(ctx, platformCli, subnetIDs, subnetSpecs); err != nil { + return err + } + + return nil +} + func (ln *localNetwork) installSubnets( ctx context.Context, subnetSpecs []network.SubnetSpec, @@ -431,7 +521,7 @@ func (ln *localNetwork) installSubnets( return nil, err } - if err = ln.addSubnetValidators(ctx, platformCli, w, subnetIDs, subnetSpecs); err != nil { + if err = ln.sendAddSubnetValidatorTxs(ctx, platformCli, w, subnetIDs, subnetSpecs); err != nil { return nil, err } @@ -541,7 +631,7 @@ func (ln *localNetwork) restartNodes( subnetIDs []ids.ID, subnetSpecs []network.SubnetSpec, validatorSpecs []network.PermissionlessValidatorSpec, - removeValidatorSpecs []network.RemoveSubnetValidatorSpec, + removeValidatorSpecs []network.SubnetValidatorSpec, nodesToRestartForBlockchainConfigUpdate set.Set[string], ) (err error) { if (subnetSpecs != nil && validatorSpecs != nil) || (subnetSpecs != nil && removeValidatorSpecs != nil) || @@ -837,7 +927,7 @@ func importPChainFromXChain(ctx context.Context, w *wallet, owner *secp256k1fx.O func (ln *localNetwork) removeSubnetValidators( ctx context.Context, - removeSubnetSpecs []network.RemoveSubnetValidatorSpec, + removeSubnetSpecs []network.SubnetValidatorSpec, ) error { ln.log.Info("removing subnet validator tx") removeSubnetSpecIDs := make([]ids.ID, len(removeSubnetSpecs)) @@ -927,6 +1017,7 @@ func (ln *localNetwork) addPermissionlessValidators( } preloadTXs[i] = subnetID } + ln.log.Info("here") w, err := newWallet(ctx, clientURI, preloadTXs) if err != nil { return err @@ -1010,7 +1101,7 @@ func (ln *localNetwork) addPermissionlessValidators( &signer.Empty{}, assetID, owner, - &secp256k1fx.OutputOwners{}, + owner, reward.PercentDenominator, common.WithContext(cctx), defaultPoll, @@ -1140,20 +1231,18 @@ func createSubnets( return subnetIDs, nil } -// add the nodes in subnet participant as validators of the given subnets, in case they are not -// the validation starts as soon as possible and its duration is as long as possible, that is, -// it ends at the time the primary network validation ends for the node -func (ln *localNetwork) addSubnetValidators( +func (ln *localNetwork) sendAddSubnetValidatorTxs( ctx context.Context, - platformCli platformvm.Client, + pCLI platformvm.Client, w *wallet, subnetIDs []ids.ID, subnetSpecs []network.SubnetSpec, ) error { - ln.log.Info(logging.Green.Wrap("adding the nodes as subnet validators")) + ln.log.Info(logging.Green.Wrap("adding subnet validators")) + for i, subnetID := range subnetIDs { cctx, cancel := createDefaultCtx(ctx) - vs, err := platformCli.GetCurrentValidators(cctx, constants.PrimaryNetworkID, nil) + vs, err := pCLI.GetCurrentValidators(cctx, constants.PrimaryNetworkID, nil) cancel() if err != nil { return err @@ -1163,7 +1252,7 @@ func (ln *localNetwork) addSubnetValidators( primaryValidatorsEndtime[v.NodeID] = time.Unix(int64(v.EndTime), 0) } cctx, cancel = createDefaultCtx(ctx) - vs, err = platformCli.GetCurrentValidators(cctx, subnetID, nil) + vs, err = pCLI.GetCurrentValidators(cctx, subnetID, nil) cancel() if err != nil { return err @@ -1180,17 +1269,17 @@ func (ln *localNetwork) addSubnetValidators( } nodeID := node.GetNodeID() if isValidator := subnetValidators.Contains(nodeID); isValidator { + ln.log.Info("already a validator", zap.String("nodeID", nodeID.String())) continue } cctx, cancel := createDefaultCtx(ctx) - txID, err := w.pWallet.IssueAddSubnetValidatorTx( + tx, err := w.pWallet.IssueAddSubnetValidatorTx( &txs.SubnetValidator{ Validator: txs.Validator{ NodeID: nodeID, - // reasonable delay in most/slow test environments - Start: uint64(time.Now().Add(validationStartOffset).Unix()), - End: uint64(primaryValidatorsEndtime[nodeID].Unix()), - Wght: subnetValidatorsWeight, + Start: uint64(time.Now().Add(validationStartOffset).Unix()), + End: uint64(primaryValidatorsEndtime[nodeID].Unix()), + Wght: subnetValidatorsWeight, }, Subnet: subnetID, }, @@ -1199,13 +1288,13 @@ func (ln *localNetwork) addSubnetValidators( ) cancel() if err != nil { - return fmt.Errorf("P-Wallet Tx Error %s %w, node ID %s, subnetID %s", "IssueAddSubnetValidatorTx", err, nodeID.String(), subnetID.String()) + return fmt.Errorf("failed to add subnet validator %s", err) } ln.log.Info("added node as a subnet validator to subnet", - zap.String("node-name", nodeName), - zap.String("node-ID", nodeID.String()), - zap.String("subnet-ID", subnetID.String()), - zap.String("tx-ID", txID.ID().String()), + zap.String("name", nodeName), + zap.String("nodeID", nodeID.String()), + zap.String("subnetID", subnetID.String()), + zap.String("tx", tx.ID().String()), ) } } @@ -1326,7 +1415,6 @@ func createBlockchainTxs( w *wallet, log logging.Logger, ) ([]*txs.Tx, error) { - fmt.Println() log.Info(logging.Green.Wrap("creating tx for each custom chain")) blockchainTxs := make([]*txs.Tx, len(chainSpecs)) for i, chainSpec := range chainSpecs { @@ -1381,7 +1469,6 @@ func (ln *localNetwork) setBlockchainConfigFiles( subnetSpecs []network.SubnetSpec, log logging.Logger, ) (set.Set[string], error) { - fmt.Println() log.Info(logging.Green.Wrap("creating config files for each custom chain")) nodesToRestart := set.Set[string]{} for i, chainSpec := range chainSpecs { diff --git a/network/network.go b/network/network.go index 383e7c89..d4d493de 100644 --- a/network/network.go +++ b/network/network.go @@ -47,7 +47,7 @@ type SubnetSpec struct { SubnetConfig []byte } -type RemoveSubnetValidatorSpec struct { +type SubnetValidatorSpec struct { NodeNames []string SubnetID string } @@ -115,7 +115,9 @@ type Network interface { // Add a validator into an elastic subnet AddPermissionlessValidators(context.Context, []PermissionlessValidatorSpec) error // Remove a validator from a subnet - RemoveSubnetValidators(context.Context, []RemoveSubnetValidatorSpec) error + RemoveSubnetValidators(context.Context, []SubnetValidatorSpec) error + // Add a validator toa subnet + AddSubnetValidators(context.Context, []SubnetValidatorSpec) error // Get the elastic subnet tx id for the given subnet id GetElasticSubnetID(context.Context, ids.ID) (ids.ID, error) } diff --git a/server/network.go b/server/network.go index 0b87a86a..63c2ca71 100644 --- a/server/network.go +++ b/server/network.go @@ -332,7 +332,44 @@ func (lc *localNetwork) AddPermissionlessValidators(ctx context.Context, validat return nil } -func (lc *localNetwork) RemoveSubnetValidator(ctx context.Context, validatorSpecs []network.RemoveSubnetValidatorSpec) error { +func (lc *localNetwork) AddSubnetValidators(ctx context.Context, validatorSpecs []network.SubnetValidatorSpec) error { + lc.lock.Lock() + defer lc.lock.Unlock() + + if len(validatorSpecs) == 0 { + ux.Print(lc.log, logging.Orange.Wrap(logging.Bold.Wrap("no validator specs provided..."))) + return nil + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + go func(ctx context.Context) { + select { + case <-lc.stopCh: + cancel() + case <-ctx.Done(): + // Nothing left to do + } + }(ctx) + + if err := lc.awaitHealthyAndUpdateNetworkInfo(ctx); err != nil { + return err + } + + if err := lc.nw.AddSubnetValidators(ctx, validatorSpecs); err != nil { + return err + } + + if err := lc.awaitHealthyAndUpdateNetworkInfo(ctx); err != nil { + return err + } + + ux.Print(lc.log, logging.Green.Wrap(logging.Bold.Wrap("subnet validators have been added to the network"))) + return nil +} + +func (lc *localNetwork) RemoveSubnetValidator(ctx context.Context, validatorSpecs []network.SubnetValidatorSpec) error { lc.lock.Lock() defer lc.lock.Unlock() diff --git a/server/server.go b/server/server.go index db39511e..32a819a3 100644 --- a/server/server.go +++ b/server/server.go @@ -522,7 +522,7 @@ func (s *server) CreateBlockchains( return &rpcpb.CreateBlockchainsResponse{ClusterInfo: clusterInfo, ChainIds: strChainIDs}, nil } -func (s *server) AddPermissionlessValidator( +func (s *server) AddValidator( _ context.Context, req *rpcpb.AddValidatorRequest, ) (*rpcpb.AddValidatorResponse, error) { @@ -533,7 +533,7 @@ func (s *server) AddPermissionlessValidator( return nil, ErrNotBootstrapped } - s.log.Debug("AddPermissionlessValidator") + s.log.Debug("AddValidator") if len(req.GetValidatorSpec()) == 0 { return nil, ErrNoValidatorSpec @@ -583,6 +583,58 @@ func (s *server) AddPermissionlessValidator( return &rpcpb.AddValidatorResponse{ClusterInfo: clusterInfo}, nil } +func (s *server) AddSubnetValidators( + _ context.Context, + req *rpcpb.AddSubnetValidatorsRequest, +) (*rpcpb.AddSubnetValidatorsResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.network == nil { + return nil, ErrNotBootstrapped + } else if len(req.GetValidatorsSpec()) == 0 { + return nil, ErrNoValidatorSpec + } + + validators := []network.SubnetValidatorSpec{} + for _, spec := range req.GetValidatorsSpec() { + validatorSpec := getSubnetValidatorSpec(spec) + validators = append(validators, validatorSpec) + } + + // check that the given subnets exist + subnets := set.Set[string]{} + subnets.Add(maps.Keys(s.clusterInfo.Subnets)...) + + for _, validatorSpec := range validators { + if validatorSpec.SubnetID == "" { + return nil, ErrNoSubnetID + } else if !subnets.Contains(validatorSpec.SubnetID) { + return nil, fmt.Errorf("subnet id %q does not exist", validatorSpec.SubnetID) + } + } + + s.clusterInfo.Healthy = false + s.clusterInfo.CustomChainsHealthy = false + + ctx, cancelFunc := context.WithTimeout(context.Background(), waitForHealthyTimeout) + defer cancelFunc() + if err := s.network.AddSubnetValidators(ctx, validators); err != nil { + s.log.Error("failed to add subnet validators", zap.Error(err)) + s.updateClusterInfo() + return nil, err + } else { + s.log.Info("successfully added subnet validators") + s.updateClusterInfo() + } + + if clusterInfo, err := deepCopy(s.clusterInfo); err != nil { + return nil, err + } else { + return &rpcpb.AddSubnetValidatorsResponse{ClusterInfo: clusterInfo}, nil + } +} + func (s *server) RemoveSubnetValidator( _ context.Context, req *rpcpb.RemoveSubnetValidatorRequest, @@ -594,26 +646,24 @@ func (s *server) RemoveSubnetValidator( return nil, ErrNotBootstrapped } - s.log.Debug("RemoveSubnetValidator") - if len(req.GetValidatorSpec()) == 0 { return nil, ErrNoValidatorSpec } - validatorSpecList := []network.RemoveSubnetValidatorSpec{} + validators := []network.SubnetValidatorSpec{} for _, spec := range req.GetValidatorSpec() { validatorSpec := getRemoveSubnetValidatorSpec(spec) - validatorSpecList = append(validatorSpecList, validatorSpec) + validators = append(validators, validatorSpec) } // check that the given subnets exist - subnetsSet := set.Set[string]{} - subnetsSet.Add(maps.Keys(s.clusterInfo.Subnets)...) + subnets := set.Set[string]{} + subnets.Add(maps.Keys(s.clusterInfo.Subnets)...) - for _, validatorSpec := range validatorSpecList { + for _, validatorSpec := range validators { if validatorSpec.SubnetID == "" { return nil, ErrNoSubnetID - } else if !subnetsSet.Contains(validatorSpec.SubnetID) { + } else if !subnets.Contains(validatorSpec.SubnetID) { return nil, fmt.Errorf("subnet id %q does not exist", validatorSpec.SubnetID) } } @@ -623,7 +673,7 @@ func (s *server) RemoveSubnetValidator( ctx, cancel := context.WithTimeout(context.Background(), waitForHealthyTimeout) defer cancel() - err := s.network.RemoveSubnetValidator(ctx, validatorSpecList) + err := s.network.RemoveSubnetValidator(ctx, validators) s.updateClusterInfo() @@ -1425,10 +1475,19 @@ func getPermissionlessValidatorSpec( return validatorSpec, nil } +func getSubnetValidatorSpec( + spec *rpcpb.SubnetValidatorsSpec, +) network.SubnetValidatorSpec { + return network.SubnetValidatorSpec{ + SubnetID: spec.SubnetId, + NodeNames: spec.GetNodeNames(), + } +} + func getRemoveSubnetValidatorSpec( spec *rpcpb.RemoveSubnetValidatorSpec, -) network.RemoveSubnetValidatorSpec { - validatorSpec := network.RemoveSubnetValidatorSpec{ +) network.SubnetValidatorSpec { + validatorSpec := network.SubnetValidatorSpec{ SubnetID: spec.SubnetId, NodeNames: spec.GetNodeNames(), }