cluster_webhook.go

// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1alpha1

import (
	"fmt"
	"reflect"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/validation/field"
	"k8s.io/apimachinery/pkg/util/version"
	ctrl "sigs.k8s.io/controller-runtime"
	logf "sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/webhook"

	"github.com/aws/eks-anywhere/pkg/features"
	"github.com/aws/eks-anywhere/pkg/semver"
)

const supportedMinorVersionIncrement int64 = 1

var devBuild, _ = semver.New("v0.0.0-dev")

// log is for logging in this package.
var clusterlog = logf.Log.WithName("cluster-resource")

func (r *Cluster) SetupWebhookWithManager(mgr ctrl.Manager) error {
	return ctrl.NewWebhookManagedBy(mgr).
		For(r).
		Complete()
}

//+kubebuilder:webhook:path=/mutate-anywhere-eks-amazonaws-com-v1alpha1-cluster,mutating=true,failurePolicy=fail,sideEffects=None,groups=anywhere.eks.amazonaws.com,resources=clusters,verbs=create;update,versions=v1alpha1,name=mutation.cluster.anywhere.amazonaws.com,admissionReviewVersions={v1,v1beta1}

var _ webhook.Defaulter = &Cluster{}

// Default implements webhook.Defaulter so a webhook will be registered for the type.
func (r *Cluster) Default() {
	clusterlog.Info("Setting up Cluster defaults", "name", r.Name, "namespace", r.Namespace)
	r.SetDefaults()
}

// Change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
//+kubebuilder:webhook:path=/validate-anywhere-eks-amazonaws-com-v1alpha1-cluster,mutating=false,failurePolicy=fail,sideEffects=None,groups=anywhere.eks.amazonaws.com,resources=clusters,verbs=create;update,versions=v1alpha1,name=validation.cluster.anywhere.amazonaws.com,admissionReviewVersions={v1,v1beta1}

var _ webhook.Validator = &Cluster{}

// ValidateCreate implements webhook.Validator so a webhook will be registered for the type.
func (r *Cluster) ValidateCreate() error {
	clusterlog.Info("validate create", "name", r.Name)

	var allErrs field.ErrorList

	if !r.IsReconcilePaused() {
		if r.IsSelfManaged() {
			return apierrors.NewBadRequest("creating new cluster on existing cluster is not supported for self managed clusters")
		} else if !features.IsActive(features.FullLifecycleAPI()) {
			return apierrors.NewBadRequest("creating new managed cluster on existing cluster is not supported")
		}
	}

	if err := r.Validate(); err != nil {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec"), r.Spec, err.Error()))
	}

	if len(allErrs) != 0 {
		return apierrors.NewInvalid(GroupVersion.WithKind(ClusterKind).GroupKind(), r.Name, allErrs)
	}

	return nil
}

// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type.
func (r *Cluster) ValidateUpdate(old runtime.Object) error {
	clusterlog.Info("validate update", "name", r.Name)
	oldCluster, ok := old.(*Cluster)
	if !ok {
		return apierrors.NewBadRequest(fmt.Sprintf("expected a Cluster but got a %T", old))
	}

	var allErrs field.ErrorList

	if r.Spec.DatacenterRef.Kind == TinkerbellDatacenterKind {
		allErrs = append(allErrs, validateUpgradeRequestTinkerbell(r, oldCluster)...)
	}

	allErrs = append(allErrs, validateImmutableFieldsCluster(r, oldCluster)...)

	allErrs = append(allErrs, validateBundlesRefCluster(r, oldCluster)...)

	allErrs = append(allErrs, ValidateKubernetesVersionSkew(r, oldCluster)...)

	allErrs = append(allErrs, validateEksaVersionCluster(r, oldCluster)...)

	allErrs = append(allErrs, ValidateEksaVersionSkew(r, oldCluster)...)

	allErrs = append(allErrs, ValidateWorkerKubernetesVersionSkew(r, oldCluster)...)

	if len(allErrs) != 0 {
		return apierrors.NewInvalid(GroupVersion.WithKind(ClusterKind).GroupKind(), r.Name, allErrs)
	}

	if err := r.Validate(); err != nil {
		allErrs = append(allErrs, field.Invalid(field.NewPath("spec"), r.Spec, err.Error()))
	}

	if len(allErrs) != 0 {
		return apierrors.NewInvalid(GroupVersion.WithKind(ClusterKind).GroupKind(), r.Name, allErrs)
	}

	return nil
}

// ValidateEksaVersionSkew ensures that upgrades are sequential by CLI minor versions.
func ValidateEksaVersionSkew(new, old *Cluster) field.ErrorList {
	var allErrs field.ErrorList
	eksaVersionPath := field.NewPath("spec").Child("EksaVersion")

	if new.Spec.EksaVersion == nil || old.Spec.EksaVersion == nil {
		return nil
	}

	// allow users to update cluster if old cluster is invalid
	oldEksaVersion, err := semver.New(string(*old.Spec.EksaVersion))
	if err != nil {
		return nil
	}

	newEksaVersion, err := semver.New(string(*new.Spec.EksaVersion))
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(eksaVersionPath, new.Spec.EksaVersion, "EksaVersion is not a valid semver"))
		return allErrs
	}

	if newEksaVersion.SamePatch(devBuild) {
		return nil
	}

	majorVersionDifference := int64(newEksaVersion.Major) - int64(oldEksaVersion.Major)
	minorVersionDifference := int64(newEksaVersion.Minor) - int64(oldEksaVersion.Minor)

	// if major different or upgrade difference greater than one minor version
	if majorVersionDifference != 0 || minorVersionDifference > supportedMinorVersionIncrement {
		allErrs = append(
			allErrs,
			field.Invalid(eksaVersionPath, new.Spec.EksaVersion, fmt.Sprintf("cannot upgrade to %v from %v: EksaVersion upgrades must have a skew of 1 minor version", newEksaVersion, oldEksaVersion)))
		return allErrs
	}

	// Allow "downgrades" if old version is greater than managment cluster. We can't check if a cluster's EksaVersion
	// is less than or equal to the mgmt cluster in the webhook. Instead we check it in the controller where the
	// EksaVersion will already be applied. However, the cluster will never begin to reconcile due to the validation.
	// We should not block users from changing EksaVersion to a lower semver in this scenario.
	failure := old.Status.FailureReason
	if failure != nil && *failure == EksaVersionInvalidReason {
		return nil
	}

	// don't allow downgrades if old version was valid
	if minorVersionDifference < 0 {
		allErrs = append(
			allErrs,
			field.Invalid(eksaVersionPath, new.Spec.EksaVersion, fmt.Sprintf("cannot downgrade from %v to %v: EksaVersion upgrades must be incremental", oldEksaVersion, newEksaVersion)))
	}

	return allErrs
}

func validateBundlesRefCluster(new, old *Cluster) field.ErrorList {
	var allErrs field.ErrorList
	bundlesRefPath := field.NewPath("spec").Child("BundlesRef")

	if old.Spec.BundlesRef != nil && new.Spec.BundlesRef == nil && new.Spec.EksaVersion == nil {
		allErrs = append(
			allErrs,
			field.Invalid(bundlesRefPath, new.Spec.BundlesRef, fmt.Sprintf("field cannot be removed after setting. Previous value %v", old.Spec.BundlesRef)))
	}

	return allErrs
}

func validateEksaVersionCluster(new, old *Cluster) field.ErrorList {
	var allErrs field.ErrorList
	eksaVersionPath := field.NewPath("spec").Child("EksaVersion")

	if old.Spec.EksaVersion != nil && new.Spec.EksaVersion == nil {
		allErrs = append(
			allErrs,
			field.Invalid(eksaVersionPath, new.Spec.EksaVersion, fmt.Sprintf("field cannot be removed after setting. Previous value %v", old.Spec.EksaVersion)))
	}

	return allErrs
}

func validateUpgradeRequestTinkerbell(new, old *Cluster) field.ErrorList {
	var allErrs field.ErrorList
	path := field.NewPath("spec")

	if old.Spec.KubernetesVersion != new.Spec.KubernetesVersion {
		if old.Spec.ControlPlaneConfiguration.Count != new.Spec.ControlPlaneConfiguration.Count {
			allErrs = append(
				allErrs,
				field.Invalid(path, new.Spec.ControlPlaneConfiguration, fmt.Sprintf("cannot perform scale up or down during rolling upgrades. Previous control plane node count %v", old.Spec.ControlPlaneConfiguration.Count)))
		}

		if len(old.Spec.WorkerNodeGroupConfigurations) != len(new.Spec.WorkerNodeGroupConfigurations) {
			allErrs = append(
				allErrs,
				field.Invalid(path, new.Spec.WorkerNodeGroupConfigurations, "cannot perform scale up or down during rolling upgrades. Please revert to the previous worker node groups."))
		}
		workerNodeGroupMap := make(map[string]*WorkerNodeGroupConfiguration)
		for i := range old.Spec.WorkerNodeGroupConfigurations {
			workerNodeGroupMap[old.Spec.WorkerNodeGroupConfigurations[i].Name] = &old.Spec.WorkerNodeGroupConfigurations[i]
		}
		for _, nodeGroupNewSpec := range new.Spec.WorkerNodeGroupConfigurations {
			workerNodeGrpOldSpec, ok := workerNodeGroupMap[nodeGroupNewSpec.Name]
			if ok && *nodeGroupNewSpec.Count != *workerNodeGrpOldSpec.Count {
				allErrs = append(
					allErrs,
					field.Invalid(path, new.Spec.WorkerNodeGroupConfigurations, fmt.Sprintf("cannot perform scale up or down during rolling upgrades. Previous worker node count %v", *workerNodeGrpOldSpec.Count)))
			}
			if !ok {
				allErrs = append(
					allErrs,
					field.Invalid(path, new.Spec.WorkerNodeGroupConfigurations, fmt.Sprintf("cannot perform scale up or down during rolling upgrades. Please remove the new worker node group %s", nodeGroupNewSpec.Name)))
			}
		}
	}

	return allErrs
}

func validateImmutableFieldsCluster(new, old *Cluster) field.ErrorList {
	if old.IsReconcilePaused() {
		return nil
	}

	var allErrs field.ErrorList
	specPath := field.NewPath("spec")

	if !old.ManagementClusterEqual(new) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("managementCluster", new.Spec.ManagementCluster.Name), fmt.Sprintf("field is immutable %v", new.Spec.ManagementCluster)))
	}

	if !new.Spec.DatacenterRef.Equal(&old.Spec.DatacenterRef) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("datacenterRef"), fmt.Sprintf("field is immutable %v", new.Spec.DatacenterRef)))
	}

	if !new.Spec.ControlPlaneConfiguration.Endpoint.Equal(old.Spec.ControlPlaneConfiguration.Endpoint, new.Spec.DatacenterRef.Kind) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("ControlPlaneConfiguration.endpoint"), fmt.Sprintf("field is immutable %v", new.Spec.ControlPlaneConfiguration.Endpoint)))
	}

	if !new.Spec.ClusterNetwork.Pods.Equal(&old.Spec.ClusterNetwork.Pods) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("clusterNetwork", "pods"), "field is immutable"))
	}

	if !new.Spec.ClusterNetwork.Services.Equal(&old.Spec.ClusterNetwork.Services) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("clusterNetwork", "services"), "field is immutable"))
	}

	if !new.Spec.ClusterNetwork.DNS.Equal(&old.Spec.ClusterNetwork.DNS) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("clusterNetwork", "dns"), "field is immutable"))
	}

	// We don't want users to be able to toggle off SkipUpgrade until we've understood the
	// implications so we are temporarily disallowing it.

	oCNI := old.Spec.ClusterNetwork.CNIConfig
	nCNI := new.Spec.ClusterNetwork.CNIConfig
	if oCNI != nil && oCNI.Cilium != nil && !oCNI.Cilium.IsManaged() && nCNI.Cilium.IsManaged() {
		allErrs = append(
			allErrs,
			field.Forbidden(
				specPath.Child("clusterNetwork", "cniConfig", "cilium", "skipUpgrade"),
				"cannot toggle off skipUpgrade once enabled",
			),
		)
	}

	if !new.Spec.ClusterNetwork.Nodes.Equal(old.Spec.ClusterNetwork.Nodes) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("clusterNetwork", "nodes"), "field is immutable"))
	}

	if !new.Spec.ProxyConfiguration.Equal(old.Spec.ProxyConfiguration) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("ProxyConfiguration"), fmt.Sprintf("field is immutable %v", new.Spec.ProxyConfiguration)))
	}

	if new.Spec.ExternalEtcdConfiguration != nil && old.Spec.ExternalEtcdConfiguration == nil {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("externalEtcdConfiguration"), "cannot switch from local to external etcd topology"),
		)
	}
	if new.Spec.ExternalEtcdConfiguration != nil && old.Spec.ExternalEtcdConfiguration != nil {
		if old.Spec.ExternalEtcdConfiguration.Count != new.Spec.ExternalEtcdConfiguration.Count {
			allErrs = append(
				allErrs,
				field.Forbidden(specPath.Child("externalEtcdConfiguration.count"), fmt.Sprintf("field is immutable %v", new.Spec.ExternalEtcdConfiguration.Count)),
			)
		}
	}

	if !new.Spec.GitOpsRef.Equal(old.Spec.GitOpsRef) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("GitOpsRef"), fmt.Sprintf("field is immutable %v", new.Spec.GitOpsRef)))
	}

	if new.Spec.DatacenterRef.Kind == TinkerbellDatacenterKind {
		if !reflect.DeepEqual(new.Spec.ControlPlaneConfiguration.Labels, old.Spec.ControlPlaneConfiguration.Labels) {
			allErrs = append(
				allErrs,
				field.Forbidden(specPath.Child("ControlPlaneConfiguration.labels"), fmt.Sprintf("field is immutable %v", new.Spec.ControlPlaneConfiguration.Labels)))
		}

		if !reflect.DeepEqual(new.Spec.ControlPlaneConfiguration.Taints, old.Spec.ControlPlaneConfiguration.Taints) {
			allErrs = append(
				allErrs,
				field.Forbidden(specPath.Child("ControlPlaneConfiguration.taints"), fmt.Sprintf("field is immutable %v", new.Spec.ControlPlaneConfiguration.Taints)))
		}

		workerNodeGroupMap := make(map[string]*WorkerNodeGroupConfiguration)
		for i := range old.Spec.WorkerNodeGroupConfigurations {
			workerNodeGroupMap[old.Spec.WorkerNodeGroupConfigurations[i].Name] = &old.Spec.WorkerNodeGroupConfigurations[i]
		}
		for _, nodeGroupNewSpec := range new.Spec.WorkerNodeGroupConfigurations {
			if workerNodeGrpOldSpec, ok := workerNodeGroupMap[nodeGroupNewSpec.Name]; ok {
				if !reflect.DeepEqual(workerNodeGrpOldSpec.Labels, nodeGroupNewSpec.Labels) {
					allErrs = append(
						allErrs,
						field.Forbidden(specPath.Child("WorkerNodeConfiguration.labels"), fmt.Sprintf("field is immutable %v", nodeGroupNewSpec.Labels)))
				}

				if !reflect.DeepEqual(workerNodeGrpOldSpec.Taints, nodeGroupNewSpec.Taints) {
					allErrs = append(
						allErrs,
						field.Forbidden(specPath.Child("WorkerNodeConfiguration.taints"), fmt.Sprintf("field is immutable %v", nodeGroupNewSpec.Taints)))
				}
			}
		}
	}

	if !old.IsSelfManaged() || features.IsActive(features.ExperimentalSelfManagedClusterUpgrade()) {
		oldAWSIamConfig, newAWSIamConfig := &Ref{}, &Ref{}
		for _, identityProvider := range new.Spec.IdentityProviderRefs {
			if identityProvider.Kind == AWSIamConfigKind {
				newAWSIamConfig = &identityProvider
				break
			}
		}

		for _, identityProvider := range old.Spec.IdentityProviderRefs {
			if identityProvider.Kind == AWSIamConfigKind {
				oldAWSIamConfig = &identityProvider
				break
			}
		}

		if !oldAWSIamConfig.Equal(newAWSIamConfig) {
			allErrs = append(
				allErrs,
				field.Forbidden(specPath.Child("identityProviderRefs", AWSIamConfigKind), fmt.Sprintf("field is immutable %v", newAWSIamConfig.Kind)))
		}
		return allErrs
	}

	clusterlog.Info("Cluster config is associated with management cluster", "name", old.Name)

	if !RefSliceEqual(new.Spec.IdentityProviderRefs, old.Spec.IdentityProviderRefs) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("IdentityProviderRefs"), fmt.Sprintf("field is immutable %v", new.Spec.IdentityProviderRefs)))
	}

	if old.Spec.KubernetesVersion != new.Spec.KubernetesVersion {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("kubernetesVersion"), fmt.Sprintf("field is immutable %v", new.Spec.KubernetesVersion)),
		)
	}

	if !old.Spec.ControlPlaneConfiguration.Equal(&new.Spec.ControlPlaneConfiguration) {
		allErrs = append(
			allErrs,
			field.Forbidden(specPath.Child("ControlPlaneConfiguration"), fmt.Sprintf("field is immutable %v", new.Spec.ControlPlaneConfiguration)))
	}

	return allErrs
}

// ValidateDelete implements webhook.Validator so a webhook will be registered for the type.
func (r *Cluster) ValidateDelete() error {
	clusterlog.Info("validate delete", "name", r.Name)

	return nil
}

// ValidateKubernetesVersionSkew validates Kubernetes version skew between upgrades.
func ValidateKubernetesVersionSkew(new, old *Cluster) field.ErrorList {
	path := field.NewPath("spec")
	oldVersion := old.Spec.KubernetesVersion
	newVersion := new.Spec.KubernetesVersion

	return validateKubeVersionSkew(newVersion, oldVersion, path)
}

func validateKubeVersionSkew(newVersion, oldVersion KubernetesVersion, path *field.Path) field.ErrorList {
	var allErrs field.ErrorList

	parsedOldVersion, err := version.ParseGeneric(string(oldVersion))
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, oldVersion, fmt.Sprintf("parsing cluster version: %v", err.Error())))
		return allErrs
	}

	parsedNewVersion, err := version.ParseGeneric(string(newVersion))
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, newVersion, fmt.Sprintf("parsing comparison version: %v", err.Error())))
		return allErrs
	}

	if parsedNewVersion.Minor() == parsedOldVersion.Minor() && parsedNewVersion.Major() == parsedOldVersion.Major() {
		return allErrs
	}

	if err := ValidateVersionSkew(parsedOldVersion, parsedNewVersion); err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, newVersion, err.Error()))
	}

	return allErrs
}

// ValidateWorkerKubernetesVersionSkew validates worker node group Kubernetes version skew between upgrades.
func ValidateWorkerKubernetesVersionSkew(new, old *Cluster) field.ErrorList {
	var allErrs field.ErrorList
	path := field.NewPath("spec").Child("WorkerNodeConfiguration.kubernetesVersion")

	newClusterVersion := new.Spec.KubernetesVersion
	oldClusterVersion := old.Spec.KubernetesVersion

	workerNodeGroupMap := make(map[string]*WorkerNodeGroupConfiguration)
	for i := range old.Spec.WorkerNodeGroupConfigurations {
		workerNodeGroupMap[old.Spec.WorkerNodeGroupConfigurations[i].Name] = &old.Spec.WorkerNodeGroupConfigurations[i]
	}
	for _, nodeGroupNewSpec := range new.Spec.WorkerNodeGroupConfigurations {
		newVersion := nodeGroupNewSpec.KubernetesVersion

		if newVersion != nil && nodeGroupNewSpec.MachineGroupRef.Kind == TinkerbellMachineConfigKind {
			allErrs = append(allErrs, field.Forbidden(path, "worker node group level kubernetesVersion is not supported for Tinkerbell"))
			return allErrs
		}

		if workerNodeGrpOldSpec, ok := workerNodeGroupMap[nodeGroupNewSpec.Name]; ok {
			oldVersion := workerNodeGrpOldSpec.KubernetesVersion
			allErrs = append(allErrs, performWorkerKubernetesValidations(oldVersion, newVersion, oldClusterVersion, newClusterVersion)...)
		} else {
			allErrs = append(allErrs, performWorkerKubernetesValidationsNewNodeGroup(newVersion, newClusterVersion)...)
		}
	}

	return allErrs
}

func performWorkerKubernetesValidationsNewNodeGroup(newVersion *KubernetesVersion, newClusterVersion KubernetesVersion) field.ErrorList {
	var allErrs field.ErrorList

	if newVersion != nil {
		allErrs = append(allErrs, validateCPWorkerKubeSkew(newClusterVersion, *newVersion)...)
	}

	return allErrs
}

func performWorkerKubernetesValidations(oldVersion, newVersion *KubernetesVersion, oldClusterVersion, newClusterVersion KubernetesVersion) field.ErrorList {
	var allErrs field.ErrorList
	path := field.NewPath("spec").Child("WorkerNodeConfiguration.kubernetesVersion")

	if oldVersion != nil && newVersion == nil {
		allErrs = append(
			allErrs,
			validateRemoveWorkerKubernetesVersion(newClusterVersion, oldClusterVersion, oldVersion)...,
		)
	}

	if oldVersion != nil && newVersion != nil {
		allErrs = append(
			allErrs,
			validateKubeVersionSkew(*newVersion, *oldVersion, path)...,
		)
	}

	if oldVersion == nil && newVersion != nil {
		allErrs = append(
			allErrs,
			validateKubeVersionSkew(*newVersion, oldClusterVersion, path)...,
		)
	}

	if newVersion != nil {
		allErrs = append(
			allErrs,
			validateCPWorkerKubeSkew(newClusterVersion, *newVersion)...,
		)
	}

	return allErrs
}

func validateRemoveWorkerKubernetesVersion(newCPVersion, oldCPVersion KubernetesVersion, oldWorkerVersion *KubernetesVersion) field.ErrorList {
	var allErrs field.ErrorList
	path := field.NewPath("spec").Child("WorkerNodeConfiguration.kubernetesVersion")

	parsedWorkerVersion, err := version.ParseGeneric(string(*oldWorkerVersion))
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, oldWorkerVersion, fmt.Sprintf("could not parse version: %v", err)))
		return allErrs
	}

	parsedOldClusterVersion, err := version.ParseGeneric(string(oldCPVersion))
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, oldCPVersion, fmt.Sprintf("could not parse version: %v", err)))
		return allErrs
	}

	parsedNewClusterVersion, err := version.ParseGeneric(string(newCPVersion))
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, newCPVersion, fmt.Sprintf("could not parse version: %v", err)))
		return allErrs
	}

	if parsedOldClusterVersion.LessThan(parsedNewClusterVersion) {
		allErrs = append(
			allErrs,
			field.Invalid(path, oldWorkerVersion, fmt.Sprintf("can't simultaneously remove worker kubernetesVersion and upgrade cluster level kubernetesVersion: %v", newCPVersion)))
		return allErrs
	}

	if err := ValidateVersionSkew(parsedWorkerVersion, parsedNewClusterVersion); err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(path, oldWorkerVersion, err.Error()))
	}

	return allErrs
}

func validKubeMinorVersionDiff(old, new KubernetesVersion) (bool, error) {
	parsedOldVersion, err := version.ParseGeneric(string(old))
	if err != nil {
		return false, fmt.Errorf("could not parse version: %v, %v", old, err)
	}

	parsedNewVersion, err := version.ParseGeneric(string(new))
	if err != nil {
		return false, fmt.Errorf("could not parse version: %v, %v", new, err)
	}

	if parsedOldVersion.Major() != parsedNewVersion.Major() {
		return false, fmt.Errorf("major versions are not the same: %v and %v", parsedOldVersion, parsedNewVersion)
	}

	oldMinor := int(parsedOldVersion.Minor())
	newMinor := int(parsedNewVersion.Minor())
	minorDiff := newMinor - oldMinor

	if minorDiff < 0 || minorDiff > 2 {
		return false, nil
	}

	return true, nil
}

func validateCPWorkerKubeSkew(cpVersion, workerVersion KubernetesVersion) field.ErrorList {
	var allErrs field.ErrorList
	workerPath := field.NewPath("spec").Child("WorkerNodeConfiguration.kubernetesVersion")
	cpPath := field.NewPath("spec").Child("kubernetesVersion")

	validSkew, err := validKubeMinorVersionDiff(workerVersion, cpVersion)
	if err != nil {
		allErrs = append(
			allErrs,
			field.Invalid(workerPath, workerVersion, fmt.Sprintf("could not determine minor version difference: %v", err.Error())))
		return allErrs
	}

	if !validSkew {
		allErrs = append(
			allErrs,
			field.Invalid(cpPath, cpVersion, fmt.Sprintf("cluster level minor version must be within 2 versions greater than worker node group version: %v", workerVersion)))
	}

	return allErrs
}