Add initial support for kubernetes 1.29 (#3048)

* Add initial support for kubernetes 1.29

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

* Update scripts fixtures

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

* Set external by default for supported cloud providers in kubernetes 1.29+

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

* Set supported versions for 1.27+

And drop all quirks and workarounds for unsupported versions

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

* Update e2e tests

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

---------

Signed-off-by: Artiom Diomin <artiom@kubermatic.com>

docs/api_reference/v1beta2.en.md

@@ -1,6 +1,6 @@
 +++
 title = "v1beta2 API Reference"
-date = 2024-02-01T11:10:05+00:00
+date = 2024-02-21T16:22:06+02:00
 weight = 11
 +++
 ## v1beta2
@@ -459,7 +459,7 @@ HostConfig describes a single control plane node.
 | bastionHostPublicKey | BastionHostPublicKey if not empty, will be used to verify bastion SSH public key | []byte | false |
 | hostname | Hostname is the hostname(1) of the host. Default value is populated at the runtime via running `hostname -f` command over ssh. | string | false |
 | isLeader | IsLeader indicates this host as a session leader. Default value is populated at the runtime. | bool | false |
-| taints | Taints are taints applied to nodes. Those taints are only applied when the node is being provisioned. If not provided (i.e. nil) for control plane nodes, it defaults to:\n  * For Kubernetes 1.23 and older: TaintEffectNoSchedule with key node-role.kubernetes.io/master\n  * For Kubernetes 1.24 and newer: TaintEffectNoSchedule with keys\n    node-role.kubernetes.io/control-plane and node-role.kubernetes.io/master\nExplicitly empty (i.e. []corev1.Taint{}) means no taints will be applied (this is default for worker nodes). | [][corev1.Taint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#taint-v1-core) | false |
+| taints | Taints are taints applied to nodes. Those taints are only applied when the node is being provisioned. If not provided (i.e. nil) for control plane nodes, it defaults to TaintEffectNoSchedule with key\n    node-role.kubernetes.io/control-plane\nExplicitly empty (i.e. []corev1.Taint{}) means no taints will be applied (this is default for worker nodes). | [][corev1.Taint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.25/#taint-v1-core) | false |
 | labels | Labels to be used to apply (or remove, with minus symbol suffix, see more kubectl help label) labels to/from node | map[string]string | false |
 | kubelet | Kubelet | [KubeletConfig](#kubeletconfig) | false |
 | operatingSystem | OperatingSystem information, can be populated at the runtime. | OperatingSystemName | false |

pkg/addons/ensure.go

@@ -337,16 +337,11 @@ func DeleteAddonByName(s *state.State, addonName string) error {
 }
 
 func ensureCSIAddons(s *state.State, addonsToDeploy []addonAction) []addonAction {
-	// We deploy available CSI drivers un-conditionally for k8s v1.23+
-	//
 	// CSIMigration, if applicable, for the cloud providers is turned on by default and requires installation of CSI drviers even if we
 	// don't use external CCM. Although mount operations would fall-back to in-tree solution if CSI driver is not available. Fallback
 	// for provision operations is NOT supported by in-tree solution.
 
-	addonsToDeploy = append(addonsToDeploy, addonAction{
-		name: resources.AddonCSIExternalSnapshotter,
-	})
-
+	var unknownProvider bool
 	switch {
 	case s.Cluster.CloudProvider.AWS != nil:
 		addonsToDeploy = append(addonsToDeploy,
@@ -426,6 +421,13 @@ func ensureCSIAddons(s *state.State, addonsToDeploy []addonAction) []addonAction
 		)
 	default:
 		s.Logger.Infof("CSI driver for %q not yet supported, skipping", s.Cluster.CloudProvider.CloudProviderName())
+		unknownProvider = true
+	}
+
+	if !unknownProvider {
+		addonsToDeploy = append(addonsToDeploy, addonAction{
+			name: resources.AddonCSIExternalSnapshotter,
+		})
 	}
 
 	return addonsToDeploy

pkg/apis/kubeone/config/testdata/config-full-v1beta1.yaml

@@ -275,10 +275,10 @@ addons:
 #     sshAgentSocket: 'env:SSH_AUTH_SOCK'
 #     # Taints is used to apply taints to the node.
 #     # If not provided defaults to TaintEffectNoSchedule, with key
-#     # node-role.kubernetes.io/master for control plane nodes.
+#     # node-role.kubernetes.io/control-plane for control plane nodes.
 #     # Explicitly empty (i.e. taints: {}) means no taints will be applied.
 #     taints:
-#     - key: "node-role.kubernetes.io/master"
+#     - key: "node-role.kubernetes.io/control-plane"
 #       effect: "NoSchedule"
 
 # A list of static workers, not managed by MachineController.

pkg/apis/kubeone/types.go

@@ -252,10 +252,8 @@ type HostConfig struct {
 	IsLeader bool `json:"isLeader,omitempty"`
 
 	// Taints are taints applied to nodes. Those taints are only applied when the node is being provisioned.
-	// If not provided (i.e. nil) for control plane nodes, it defaults to:
-	//   * For Kubernetes 1.23 and older: TaintEffectNoSchedule with key node-role.kubernetes.io/master
-	//   * For Kubernetes 1.24 and newer: TaintEffectNoSchedule with keys
-	//     node-role.kubernetes.io/control-plane and node-role.kubernetes.io/master
+	// If not provided (i.e. nil) for control plane nodes, it defaults to TaintEffectNoSchedule with key
+	//     node-role.kubernetes.io/control-plane
 	// Explicitly empty (i.e. []corev1.Taint{}) means no taints will be applied (this is default for worker nodes).
 	Taints []corev1.Taint `json:"taints,omitempty"`
 

pkg/apis/kubeone/v1beta2/defaults.go

@@ -60,6 +60,7 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
 
 func SetDefaults_KubeOneCluster(obj *KubeOneCluster) {
 	SetDefaults_Hosts(obj)
+	SetDefaults_CloudProvider(obj)
 	SetDefaults_APIEndpoints(obj)
 	SetDefaults_Versions(obj)
 	SetDefaults_ContainerRuntime(obj)
@@ -73,6 +74,18 @@ func SetDefaults_KubeOneCluster(obj *KubeOneCluster) {
 	SetDefaults_CloudConfig(obj)
 }
 
+func SetDefaults_CloudProvider(obj *KubeOneCluster) {
+	gteKube129Condition, _ := semver.NewConstraint(">= 1.29")
+	actualVer, err := semver.NewVersion(obj.Versions.Kubernetes)
+	if err != nil {
+		return
+	}
+
+	if gteKube129Condition.Check(actualVer) && obj.CloudProvider.None == nil {
+		obj.CloudProvider.External = true
+	}
+}
+
 func SetDefaults_CloudConfig(obj *KubeOneCluster) {
 	if obj.CloudProvider.AWS != nil && obj.CloudProvider.External {
 		if obj.CloudProvider.CloudConfig == "" {
@@ -89,13 +102,6 @@ func SetDefaults_Hosts(obj *KubeOneCluster) {
 
 	setDefaultLeader := true
 
-	gteKube124Condition, _ := semver.NewConstraint(">= 1.24")
-	ltKube125Condition, _ := semver.NewConstraint("< 1.25")
-	actualVer, err := semver.NewVersion(obj.Versions.Kubernetes)
-	if err != nil {
-		return
-	}
-
 	// Define a unique ID for each host
 	for idx := range obj.ControlPlane.Hosts {
 		if setDefaultLeader && obj.ControlPlane.Hosts[idx].IsLeader {
@@ -106,20 +112,10 @@ func SetDefaults_Hosts(obj *KubeOneCluster) {
 		obj.ControlPlane.Hosts[idx].ID = idx
 		defaultHostConfig(&obj.ControlPlane.Hosts[idx])
 		if obj.ControlPlane.Hosts[idx].Taints == nil {
-			if ltKube125Condition.Check(actualVer) {
-				obj.ControlPlane.Hosts[idx].Taints = []corev1.Taint{
-					{
-						Effect: corev1.TaintEffectNoSchedule,
-						Key:    "node-role.kubernetes.io/master",
-					},
-				}
-			}
-			if gteKube124Condition.Check(actualVer) {
-				obj.ControlPlane.Hosts[idx].Taints = append(obj.ControlPlane.Hosts[idx].Taints, corev1.Taint{
-					Effect: corev1.TaintEffectNoSchedule,
-					Key:    "node-role.kubernetes.io/control-plane",
-				})
-			}
+			obj.ControlPlane.Hosts[idx].Taints = append(obj.ControlPlane.Hosts[idx].Taints, corev1.Taint{
+				Effect: corev1.TaintEffectNoSchedule,
+				Key:    "node-role.kubernetes.io/control-plane",
+			})
 		}
 	}
 	if setDefaultLeader {
@@ -231,7 +227,7 @@ func SetDefaults_Proxy(obj *KubeOneCluster) {
 func SetDefaults_MachineController(obj *KubeOneCluster) {
 	if obj.MachineController == nil {
 		obj.MachineController = &MachineControllerConfig{
-			Deploy: true,
+			Deploy: obj.CloudProvider.None == nil,
 		}
 	}
 }

pkg/apis/kubeone/v1beta2/types.go

@@ -258,10 +258,8 @@ type HostConfig struct {
 	IsLeader bool `json:"isLeader,omitempty"`
 
 	// Taints are taints applied to nodes. Those taints are only applied when the node is being provisioned.
-	// If not provided (i.e. nil) for control plane nodes, it defaults to:
-	//   * For Kubernetes 1.23 and older: TaintEffectNoSchedule with key node-role.kubernetes.io/master
-	//   * For Kubernetes 1.24 and newer: TaintEffectNoSchedule with keys
-	//     node-role.kubernetes.io/control-plane and node-role.kubernetes.io/master
+	// If not provided (i.e. nil) for control plane nodes, it defaults to TaintEffectNoSchedule with key
+	//     node-role.kubernetes.io/control-plane
 	// Explicitly empty (i.e. []corev1.Taint{}) means no taints will be applied (this is default for worker nodes).
 	Taints []corev1.Taint `json:"taints,omitempty"`
 

pkg/apis/kubeone/validation/validation.go

@@ -39,32 +39,29 @@ import (
 
 const (
 	// lowerVersionConstraint defines a semver constraint that validates Kubernetes versions against a lower bound
-	lowerVersionConstraint = ">= 1.25"
+	lowerVersionConstraint = ">= 1.27"
 	// upperVersionConstraint defines a semver constraint that validates Kubernetes versions against an upper bound
-	upperVersionConstraint = "<= 1.28"
-	// gte125VersionConstraint defines a semver constraint that validates Kubernetes versions >= 1.25
-	gte125VersionConstraint = ">= 1.25"
+	upperVersionConstraint = "<= 1.29"
 )
 
 var (
-	lowerConstraint  = semverutil.MustParseConstraint(lowerVersionConstraint)
-	upperConstraint  = semverutil.MustParseConstraint(upperVersionConstraint)
-	gte125Constraint = semverutil.MustParseConstraint(gte125VersionConstraint)
+	lowerConstraint = semverutil.MustParseConstraint(lowerVersionConstraint)
+	upperConstraint = semverutil.MustParseConstraint(upperVersionConstraint)
 )
 
 // ValidateKubeOneCluster validates the KubeOneCluster object
 func ValidateKubeOneCluster(c kubeoneapi.KubeOneCluster) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	allErrs = append(allErrs, ValidateName(c.Name, field.NewPath("name"))...)
-	allErrs = append(allErrs, ValidateControlPlaneConfig(c.ControlPlane, c.Versions, c.ClusterNetwork, field.NewPath("controlPlane"))...)
+	allErrs = append(allErrs, ValidateControlPlaneConfig(c.ControlPlane, c.ClusterNetwork, field.NewPath("controlPlane"))...)
 	allErrs = append(allErrs, ValidateAPIEndpoint(c.APIEndpoint, field.NewPath("apiEndpoint"))...)
 	allErrs = append(allErrs, ValidateCloudProviderSpec(c.CloudProvider, c.ClusterNetwork, field.NewPath("provider"))...)
 	allErrs = append(allErrs, ValidateVersionConfig(c.Versions, field.NewPath("versions"))...)
 	allErrs = append(allErrs, ValidateKubernetesSupport(c, field.NewPath(""))...)
 	allErrs = append(allErrs, ValidateContainerRuntimeConfig(c.ContainerRuntime, c.Versions, field.NewPath("containerRuntime"))...)
 	allErrs = append(allErrs, ValidateClusterNetworkConfig(c.ClusterNetwork, c.CloudProvider, field.NewPath("clusterNetwork"))...)
-	allErrs = append(allErrs, ValidateStaticWorkersConfig(c.StaticWorkers, c.Versions, c.ClusterNetwork, field.NewPath("staticWorkers"))...)
+	allErrs = append(allErrs, ValidateStaticWorkersConfig(c.StaticWorkers, c.ClusterNetwork, field.NewPath("staticWorkers"))...)
 
 	if c.MachineController != nil && c.MachineController.Deploy {
 		allErrs = append(allErrs, ValidateDynamicWorkerConfig(c.DynamicWorkers, c.CloudProvider, field.NewPath("dynamicWorkers"))...)
@@ -128,11 +125,11 @@ func ValidateName(name string, fldPath *field.Path) field.ErrorList {
 }
 
 // ValidateControlPlaneConfig validates the ControlPlaneConfig structure
-func ValidateControlPlaneConfig(c kubeoneapi.ControlPlaneConfig, version kubeoneapi.VersionConfig, clusterNetwork kubeoneapi.ClusterNetworkConfig, fldPath *field.Path) field.ErrorList {
+func ValidateControlPlaneConfig(c kubeoneapi.ControlPlaneConfig, clusterNetwork kubeoneapi.ClusterNetworkConfig, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	if len(c.Hosts) > 0 {
-		allErrs = append(allErrs, ValidateHostConfig(c.Hosts, version, clusterNetwork, fldPath.Child("hosts"))...)
+		allErrs = append(allErrs, ValidateHostConfig(c.Hosts, clusterNetwork, fldPath.Child("hosts"))...)
 	} else {
 		allErrs = append(allErrs, field.Invalid(fldPath.Child("hosts"), "",
 			".controlPlane.Hosts is a required field. There must be at least one control plane instance in the cluster."))
@@ -533,11 +530,11 @@ func ValidateCNI(c *kubeoneapi.CNI, fldPath *field.Path) field.ErrorList {
 }
 
 // ValidateStaticWorkersConfig validates the StaticWorkersConfig structure
-func ValidateStaticWorkersConfig(staticWorkers kubeoneapi.StaticWorkersConfig, version kubeoneapi.VersionConfig, clusterNetwork kubeoneapi.ClusterNetworkConfig, fldPath *field.Path) field.ErrorList {
+func ValidateStaticWorkersConfig(staticWorkers kubeoneapi.StaticWorkersConfig, clusterNetwork kubeoneapi.ClusterNetworkConfig, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	if len(staticWorkers.Hosts) > 0 {
-		allErrs = append(allErrs, ValidateHostConfig(staticWorkers.Hosts, version, clusterNetwork, fldPath.Child("hosts"))...)
+		allErrs = append(allErrs, ValidateHostConfig(staticWorkers.Hosts, clusterNetwork, fldPath.Child("hosts"))...)
 	}
 
 	return allErrs
@@ -731,59 +728,45 @@ func ValidateHelmReleases(helmReleases []kubeoneapi.HelmRelease, fldPath *field.
 }
 
 // ValidateHostConfig validates the HostConfig structure
-func ValidateHostConfig(hosts []kubeoneapi.HostConfig, version kubeoneapi.VersionConfig, clusterNetwork kubeoneapi.ClusterNetworkConfig, fldPath *field.Path) field.ErrorList {
+func ValidateHostConfig(hosts []kubeoneapi.HostConfig, clusterNetwork kubeoneapi.ClusterNetworkConfig, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
-	v, err := semver.NewVersion(version.Kubernetes)
-	if err != nil {
-		allErrs = append(allErrs, field.Invalid(fldPath.Child("kubernetes"), version, ".versions.kubernetes is not a semver string"))
-
-		return allErrs
-	}
-
 	leaderFound := false
-	for _, h := range hosts {
-		if leaderFound && h.IsLeader {
-			allErrs = append(allErrs, field.Invalid(fldPath, h.IsLeader, "only one leader is allowed"))
+	for _, host := range hosts {
+		if leaderFound && host.IsLeader {
+			allErrs = append(allErrs, field.Invalid(fldPath, host.IsLeader, "only one leader is allowed"))
 		}
-		if h.IsLeader {
+		if host.IsLeader {
 			leaderFound = true
 		}
-		if len(h.PublicAddress) == 0 {
+		if len(host.PublicAddress) == 0 {
 			allErrs = append(allErrs, field.Required(fldPath, "no public IP/address given"))
 		}
 
-		if (clusterNetwork.IPFamily == kubeoneapi.IPFamilyIPv6 || clusterNetwork.IPFamily == kubeoneapi.IPFamilyIPv4IPv6 || clusterNetwork.IPFamily == kubeoneapi.IPFamilyIPv6IPv4) && len(h.IPv6Addresses) == 0 {
+		if (clusterNetwork.IPFamily == kubeoneapi.IPFamilyIPv6 || clusterNetwork.IPFamily == kubeoneapi.IPFamilyIPv4IPv6 || clusterNetwork.IPFamily == kubeoneapi.IPFamilyIPv6IPv4) && len(host.IPv6Addresses) == 0 {
 			allErrs = append(allErrs, field.Required(fldPath, "no IPv6 address given"))
 		}
-		if len(h.PrivateAddress) == 0 {
+		if len(host.PrivateAddress) == 0 {
 			allErrs = append(allErrs, field.Required(fldPath, "no private IP/address givevn"))
 		}
-		if len(h.SSHPrivateKeyFile) == 0 && len(h.SSHAgentSocket) == 0 {
-			allErrs = append(allErrs, field.Invalid(fldPath, h.SSHPrivateKeyFile, "neither SSH private key nor agent socket given, don't know how to authenticate"))
-			allErrs = append(allErrs, field.Invalid(fldPath, h.SSHAgentSocket, "neither SSH private key nor agent socket given, don't know how to authenticate"))
+		if len(host.SSHPrivateKeyFile) == 0 && len(host.SSHAgentSocket) == 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath, host.SSHPrivateKeyFile, "neither SSH private key nor agent socket given, don't know how to authenticate"))
+			allErrs = append(allErrs, field.Invalid(fldPath, host.SSHAgentSocket, "neither SSH private key nor agent socket given, don't know how to authenticate"))
 		}
-		if len(h.SSHUsername) == 0 {
+		if len(host.SSHUsername) == 0 {
 			allErrs = append(allErrs, field.Required(fldPath, "no SSH username given"))
 		}
-		if !h.OperatingSystem.IsValid() {
-			allErrs = append(allErrs, field.Invalid(fldPath.Child("operatingSystem"), h.OperatingSystem, "invalid operatingSystem provided"))
+		if !host.OperatingSystem.IsValid() {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("operatingSystem"), host.OperatingSystem, "invalid operatingSystem provided"))
 		}
-		if h.Kubelet.MaxPods != nil && *h.Kubelet.MaxPods <= 0 {
-			allErrs = append(allErrs, field.Invalid(fldPath.Child("kubelet").Child("maxPods"), h.Kubelet.MaxPods, "maxPods must be a positive number"))
+		if host.Kubelet.MaxPods != nil && *host.Kubelet.MaxPods <= 0 {
+			allErrs = append(allErrs, field.Invalid(fldPath.Child("kubelet").Child("maxPods"), host.Kubelet.MaxPods, "maxPods must be a positive number"))
 		}
-		for labelKey, labelValue := range h.Labels {
+		for labelKey, labelValue := range host.Labels {
 			if strings.HasSuffix(labelKey, "-") && labelValue != "" {
 				allErrs = append(allErrs, field.Invalid(fldPath.Child("labels"), labelValue, "label to remove cannot have value"))
 			}
 		}
-		if gte125Constraint.Check(v) {
-			for _, taint := range h.Taints {
-				if taint.Key == "node-role.kubernetes.io/master" {
-					allErrs = append(allErrs, field.Forbidden(fldPath.Child("taints"), fmt.Sprintf("%q taint is forbidden for clusters running Kubernetes 1.25+", "node-role.kubernetes.io/master")))
-				}
-			}
-		}
 	}
 
 	return allErrs