feat(platform): update cluster upgrade process (#867)

api/platform/types.go

@@ -120,9 +120,6 @@ type ClusterSpec struct {
 	// Etcd holds configuration for etcd.
 	// +optional
 	Etcd *Etcd
-	// Upgrade control upgrade process.
-	// +optional
-	Upgrade Upgrade
 	// If true will use hostname as nodename, if false will use machine IP as nodename.
 	// +optional
 	HostnameAsNodename bool
@@ -359,6 +356,9 @@ type ClusterFeature struct {
 	EnableMetricsServer bool
 	// +optional
 	IPv6DualStack bool
+	// Upgrade control upgrade process.
+	// +optional
+	Upgrade Upgrade
 }
 
 type HA struct {
@@ -491,6 +491,11 @@ type UpgradeStrategy struct {
 	// 100% means ignore any pods unready which may be used in one worker node, use this carefully!
 	// default value is 0%.
 	MaxUnready intstr.IntOrString
+	// Whether drain node before upgrade.
+	// Draining node before upgrade is recommended.
+	// But not all pod running as cows, a few running as pets.
+	// If your pod can not accept be expelled from current node, this value should be false.
+	DrainNodeBeforeUpgrade bool
 }
 
 // ResourceList is a set of (resource name, quantity) pairs.

api/platform/v1/defaults.go

@@ -28,12 +28,12 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
 }
 
 func SetDefaults_ClusterSpec(obj *ClusterSpec) {
-	if obj.Upgrade.Mode == "" {
-		obj.Upgrade.Mode = UpgradeModeAuto
+	if obj.Features.Upgrade.Mode == "" {
+		obj.Features.Upgrade.Mode = UpgradeModeAuto
 	}
-	if obj.Upgrade.Strategy.MaxUnready == nil {
+	if obj.Features.Upgrade.Strategy.MaxUnready == nil {
 		maxUnready := intstr.FromInt(0)
-		obj.Upgrade.Strategy.MaxUnready = &maxUnready
+		obj.Features.Upgrade.Strategy.MaxUnready = &maxUnready
 	}
 }
 

api/platform/v1/field_constants.go

@@ -0,0 +1,7 @@
+package v1
+
+// Field path constants that are specific to the internal API
+// representation.
+const (
+	MachineClusterField = "spec.clusterName"
+)

api/platform/v1/types.go

@@ -124,9 +124,6 @@ type ClusterSpec struct {
 	// Etcd holds configuration for etcd.
 	// +optional
 	Etcd *Etcd `json:"etcd,omitempty" protobuf:"bytes,21,opt,name=etcd"`
-	// Upgrade control upgrade process.
-	// +optional
-	Upgrade Upgrade `json:"upgrade,omitempty" protobuf:"bytes,22,opt,name=upgrade"`
 	// If true will use hostname as nodename, if false will use machine IP as nodename.
 	// +optional
 	HostnameAsNodename bool `json:"hostnameAsNodename,omitempty" protobuf:"bytes,23,opt,name=hostnameAsNodename"`
@@ -365,6 +362,9 @@ type ClusterFeature struct {
 	EnableMetricsServer bool `json:"enableMetricsServer,omitempty" protobuf:"bytes,12,opt,name=enableMetricsServer"`
 	// +optional
 	IPv6DualStack bool `json:"ipv6DualStack,omitempty" protobuf:"bytes,13,opt,name=ipv6DualStack"`
+	// Upgrade control upgrade process.
+	// +optional
+	Upgrade Upgrade `json:"upgrade,omitempty" protobuf:"bytes,22,opt,name=upgrade"`
 }
 
 type HA struct {
@@ -505,6 +505,12 @@ type UpgradeStrategy struct {
 	// default value is 0%.
 	// +optional
 	MaxUnready *intstr.IntOrString `json:"maxUnready,omitempty" protobuf:"bytes,1,opt,name=maxUnready"`
+	// Whether drain node before upgrade.
+	// Draining node before upgrade is recommended.
+	// But not all pod running as cows, a few running as pets.
+	// If your pod can not accept be expelled from current node, this value should be false.
+	// +optional
+	DrainNodeBeforeUpgrade *bool `json:"drainNodeBeforeUpgrade,omitempty" protobuf:"varint,2,opt,name=drainNodeBeforeUpgrade"`
 }
 
 // ResourceList is a set of (resource name, quantity) pairs.

pkg/platform/provider/baremetal/cluster/update.go

@@ -22,10 +22,13 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
 	"github.com/thoas/go-funk"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	certutil "k8s.io/client-go/util/cert"
 	platformv1 "tkestack.io/tke/api/platform/v1"
 	"tkestack.io/tke/pkg/platform/provider/baremetal/constants"
@@ -137,14 +140,31 @@ func (p *Provider) EnsurePreClusterUpgradeHook(ctx context.Context, c *v1.Cluste
 }
 
 func (p *Provider) EnsureUpgradeControlPlaneNode(ctx context.Context, c *v1.Cluster) error {
+	// check all machines are upgraded before upgrade cluster
+	machines, err := p.platformClient.Machines().List(context.TODO(), metav1.ListOptions{
+		LabelSelector: fields.OneTermEqualSelector(constants.LabelNodeNeedUpgrade, "").String(),
+		FieldSelector: fields.OneTermEqualSelector(platformv1.MachineClusterField, c.Name).String(),
+	})
+	if err != nil {
+		return err
+	}
+	if len(machines.Items) != 0 {
+		var itemsName []string
+		for _, item := range machines.Items {
+			itemsName = append(itemsName, item.Name)
+		}
+		return fmt.Errorf("some machines, [%s], need to be upgraded", strings.Join(itemsName, ","))
+	}
+
 	client, err := c.Clientset()
 	if err != nil {
 		return err
 	}
 	option := kubeadm.UpgradeOption{
-		NodeRole:   kubeadm.NodeRoleMaster,
-		Version:    c.Spec.Version,
-		MaxUnready: c.Spec.Upgrade.Strategy.MaxUnready,
+		NodeRole:               kubeadm.NodeRoleMaster,
+		Version:                c.Spec.Version,
+		MaxUnready:             c.Spec.Features.Upgrade.Strategy.MaxUnready,
+		DrainNodeBeforeUpgrade: c.Spec.Features.Upgrade.Strategy.DrainNodeBeforeUpgrade,
 	}
 	for i, machine := range c.Spec.Machines {
 		option.MachineName = machine.Username
@@ -154,21 +174,24 @@ func (p *Provider) EnsureUpgradeControlPlaneNode(ctx context.Context, c *v1.Clus
 		if err != nil {
 			return err
 		}
-		upgraded, err := kubeadm.UpgradeNode(s, client, p.platformClient, option)
+		upgraded, err := kubeadm.UpgradeNode(s, client, p.platformClient, c, option)
 		if err != nil {
 			return err
 		}
 
-		// Label next node when upgraded all master nodes and upgrade mode is auto.
-		if upgraded && c.Spec.Upgrade.Mode == platformv1.UpgradeModeAuto && i == len(c.Spec.Machines)-1 {
-			err = kubeadm.MarkNextUpgradeWorkerNode(client, p.platformClient, option.Version)
-			if err != nil {
+		if i == len(c.Spec.Machines)-1 && upgraded {
+			if err := kubeadm.AddNeedUpgradeLabel(p.platformClient, c.Name); err != nil {
 				return err
 			}
+			// Label next node when upgraded all master nodes and upgrade mode is auto.
+			if c.Spec.Features.Upgrade.Mode == platformv1.UpgradeModeAuto {
+				err = kubeadm.MarkNextUpgradeWorkerNode(client, p.platformClient, option.Version, c.Name)
+				if err != nil {
+					return err
+				}
+			}
 		}
 	}
-	c.Status.Phase = platformv1.ClusterRunning
-
 	return nil
 }