Merge pull request #259 from rancher/enhancement/create-node-wait

[Enhancement] createNode: add --wait and --timeout flags

cmd/create/createNode.go

@@ -23,6 +23,7 @@ package create
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/spf13/cobra"
 
@@ -36,6 +37,8 @@ import (
 // NewCmdCreateNode returns a new cobra command
 func NewCmdCreateNode() *cobra.Command {
 
+	createNodeOpts := k3d.CreateNodeOpts{}
+
 	// create new command
 	cmd := &cobra.Command{
 		Use:   "node NAME",
@@ -44,11 +47,9 @@ func NewCmdCreateNode() *cobra.Command {
 		Args:  cobra.ExactArgs(1), // exactly one name accepted // TODO: if not specified, inherit from cluster that the node shall belong to, if that is specified
 		Run: func(cmd *cobra.Command, args []string) {
 			nodes, cluster := parseCreateNodeCmd(cmd, args)
-			for _, node := range nodes {
-				if err := k3dc.AddNodeToCluster(cmd.Context(), runtimes.SelectedRuntime, node, cluster); err != nil {
-					log.Errorf("Failed to add node '%s' to cluster '%s'", node.Name, cluster.Name)
-					log.Errorln(err)
-				}
+			if err := k3dc.AddNodesToCluster(cmd.Context(), runtimes.SelectedRuntime, nodes, cluster, createNodeOpts); err != nil {
+				log.Errorf("Failed to add nodes '%+v' to cluster '%s'", nodes, cluster.Name)
+				log.Errorln(err)
 			}
 		},
 	}
@@ -63,6 +64,9 @@ func NewCmdCreateNode() *cobra.Command {
 
 	cmd.Flags().StringP("image", "i", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.GetK3sVersion(false)), "Specify k3s image used for the node(s)")
 
+	cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.")
+	cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.")
+
 	// done
 	return cmd
 }

docs/usage/commands.md

@@ -25,6 +25,8 @@ k3d
       -i, --image  # specify which k3s image should be used for the node(s)
           --replicas  # specify how many replicas you want to create with this spec
           --role  # specify the node role
+      --wait  # wait for the node to be up and running before returning
+      --timeout # specify a timeout duration, after which the node creation will be interrupted, if not done yet
   delete
     cluster CLUSTERNAME  # delete an existing cluster
       -a, --all  # delete all existing clusters

pkg/cluster/cluster.go

@@ -155,7 +155,7 @@ func CreateCluster(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 
 		// create node
 		log.Infof("Creating node '%s'", node.Name)
-		if err := CreateNode(ctx, runtime, node); err != nil {
+		if err := CreateNode(ctx, runtime, node, k3d.CreateNodeOpts{}); err != nil {
 			log.Errorln("Failed to create node")
 			return err
 		}
@@ -301,7 +301,7 @@ func CreateCluster(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 			}
 			cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
 			log.Infof("Creating LoadBalancer '%s'", lbNode.Name)
-			if err := CreateNode(ctx, runtime, lbNode); err != nil {
+			if err := CreateNode(ctx, runtime, lbNode, k3d.CreateNodeOpts{}); err != nil {
 				log.Errorln("Failed to create loadbalancer")
 				return err
 			}

pkg/cluster/node.go

@@ -33,10 +33,11 @@ import (
 	"github.com/rancher/k3d/pkg/runtimes"
 	k3d "github.com/rancher/k3d/pkg/types"
 	log "github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 )
 
 // AddNodeToCluster adds a node to an existing cluster
-func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, cluster *k3d.Cluster) error {
+func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, cluster *k3d.Cluster, createNodeOpts k3d.CreateNodeOpts) error {
 	cluster, err := GetCluster(ctx, runtime, cluster)
 	if err != nil {
 		log.Errorf("Failed to find specified cluster '%s'", cluster.Name)
@@ -126,7 +127,7 @@ func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 		}
 	}
 
-	if err := CreateNode(ctx, runtime, node); err != nil {
+	if err := CreateNode(ctx, runtime, node, k3d.CreateNodeOpts{}); err != nil {
 		return err
 	}
 
@@ -141,17 +142,70 @@ func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 	return nil
 }
 
+// AddNodesToCluster adds multiple nodes to a chosen cluster
+func AddNodesToCluster(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, cluster *k3d.Cluster, createNodeOpts k3d.CreateNodeOpts) error {
+	if createNodeOpts.Timeout > 0*time.Second {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, createNodeOpts.Timeout)
+		defer cancel()
+	}
+
+	nodeWaitGroup, ctx := errgroup.WithContext(ctx)
+	for _, node := range nodes {
+		if err := AddNodeToCluster(ctx, runtime, node, cluster, k3d.CreateNodeOpts{}); err != nil {
+			return err
+		}
+		if createNodeOpts.Wait {
+			currentNode := node
+			nodeWaitGroup.Go(func() error {
+				log.Debugf("Starting to wait for node '%s'", currentNode.Name)
+				return WaitForNodeLogMessage(ctx, runtime, currentNode, k3d.ReadyLogMessageByRole[currentNode.Role], time.Time{})
+			})
+		}
+	}
+	if err := nodeWaitGroup.Wait(); err != nil {
+		log.Errorln("Failed to bring up all nodes in time. Check the logs:")
+		log.Errorf(">>> %+v", err)
+		return fmt.Errorf("Failed to add nodes")
+	}
+
+	return nil
+}
+
 // CreateNodes creates a list of nodes
-func CreateNodes(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node) { // TODO: pass `--atomic` flag, so we stop and return an error if any node creation fails?
+func CreateNodes(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, createNodeOpts k3d.CreateNodeOpts) error { // TODO: pass `--atomic` flag, so we stop and return an error if any node creation fails?
+	if createNodeOpts.Timeout > 0*time.Second {
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, createNodeOpts.Timeout)
+		defer cancel()
+	}
+
+	nodeWaitGroup, ctx := errgroup.WithContext(ctx)
 	for _, node := range nodes {
-		if err := CreateNode(ctx, runtime, node); err != nil {
+		if err := CreateNode(ctx, runtime, node, k3d.CreateNodeOpts{}); err != nil {
 			log.Error(err)
 		}
+		if createNodeOpts.Wait {
+			currentNode := node
+			nodeWaitGroup.Go(func() error {
+				log.Debugf("Starting to wait for node '%s'", currentNode.Name)
+				return WaitForNodeLogMessage(ctx, runtime, currentNode, k3d.ReadyLogMessageByRole[currentNode.Role], time.Time{})
+			})
+		}
 	}
+
+	if err := nodeWaitGroup.Wait(); err != nil {
+		log.Errorln("Failed to bring up all nodes in time. Check the logs:")
+		log.Errorf(">>> %+v", err)
+		return fmt.Errorf("Failed to create nodes")
+	}
+
+	return nil
+
 }
 
 // CreateNode creates a new containerized k3s node
-func CreateNode(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node) error {
+func CreateNode(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, createNodeOpts k3d.CreateNodeOpts) error {
 	log.Debugf("Creating node from spec\n%+v", node)
 
 	/*

pkg/types/types.go

@@ -47,7 +47,7 @@ const DefaultObjectNamePrefix = "k3d"
 // ReadyLogMessageMaster defines the log messages we wait for until a master node is considered ready
 var ReadyLogMessageByRole = map[Role]string{
 	MasterRole:       "Wrote kubeconfig",
-	WorkerRole:       "",
+	WorkerRole:       "Successfully registered node",
 	LoadBalancerRole: "start worker processes",
 }
 
@@ -130,6 +130,18 @@ type StartClusterOpts struct {
 	Timeout       time.Duration
 }
 
+// CreateNodeOpts describes a set of options one can set when creating a new node
+type CreateNodeOpts struct {
+	Wait    bool
+	Timeout time.Duration
+}
+
+// StartNodeOpts describes a set of options one can set when (re-)starting a node
+type StartNodeOpts struct {
+	Wait    bool
+	Timeout time.Duration
+}
+
 // ClusterNetwork describes a network which a cluster is running in
 type ClusterNetwork struct {
 	Name     string `yaml:"name" json:"name,omitempty"`

tests/test_full_lifecycle.sh

@@ -39,14 +39,11 @@ info "Checking that we have 2 nodes online..."
 check_multi_node "$clustername" 2 || failed "failed to verify number of nodes"
 
 # 4. adding another worker node
-# info "Adding one worker node..."
-# LOG_LEVEL=debug $EXE create node "extra-worker" --cluster "$clustername" --role "worker" || failed "failed to add worker node"
-#
-# info "Waiting for a bit to give the new node enough time to boot and register..."
-# sleep 10
-#
-# info "Checking that we have 3 nodes available now..."
-# check_multi_node "$clustername" 3 || failed "failed to verify number of nodes"
+info "Adding one worker node..."
+LOG_LEVEL=debug $EXE create node "extra-worker" --cluster "$clustername" --role "worker" --wait --timeout 360s || failed "failed to add worker node"
+
+info "Checking that we have 3 nodes available now..."
+check_multi_node "$clustername" 3 || failed "failed to verify number of nodes"
 
 # 4. load an image into the cluster
 info "Loading an image into the cluster..."