Skip to content

✨ Implement core logic for chained upgrades #12726

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Sep 12, 2025
Merged

✨ Implement core logic for chained upgrades #12726

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
673e556
Chained upgrades
fabriziopandini Sep 1, 2025
669e1d5
Fix review findings
sbueringer Sep 9, 2025
34771f2
Merge pull request #312 from sbueringer/pr-chained-upgrades-fix-some-…
fabriziopandini Sep 10, 2025
5a176a9
Address comments
fabriziopandini Sep 11, 2025
f396e46
Fix some minor findings
sbueringer Sep 12, 2025
809e203
Merge pull request #313 from sbueringer/pr-chained-upgrades-fix-some-…
fabriziopandini Sep 12, 2025
1c4c680
Temporarily drop race from test-infrastructure
fabriziopandini Sep 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -938,30 +938,31 @@ test-cover: ## Run unit and integration tests and generate a coverage report
go tool cover -func=out/coverage.out -o out/coverage.txt
go tool cover -html=out/coverage.out -o out/coverage.html

.PHONY: test-docker-infrastructure
test-docker-infrastructure: $(SETUP_ENVTEST) ## Run unit and integration tests with race detector for docker infrastructure provider
.PHONY: test-infrastructure
test-infrastructure: $(SETUP_ENVTEST) ## Run unit and integration tests with race detector for docker infrastructure provider
# Note: Fuzz tests are not executed with race detector because they would just time out.
# To achieve that, all files with fuzz tests have the "!race" build tag, to still run fuzz tests
# we have an additional `go test` run that focuses on "TestFuzzyConversion".
cd $(CAPD_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race ./... $(TEST_ARGS)
$(MAKE) test-docker-infrastructure-conversions TEST_ARGS="$(TEST_ARGS)"
cd test/infrastructure; KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS)
$(MAKE) test-infrastructure-conversions TEST_ARGS="$(TEST_ARGS)"

.PHONY: test-docker-infrastructure-conversions
test-docker-infrastructure-conversions: $(SETUP_ENVTEST) ## Run conversions test for docker infrastructure provider
cd $(CAPD_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" ./... $(TEST_ARGS)
.PHONY: test-infrastructure-conversions
test-infrastructure-conversions: $(SETUP_ENVTEST) ## Run conversions test for docker infrastructure provider
cd test/infrastructure; KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" ./... $(TEST_ARGS)

.PHONY: test-docker-infrastructure-verbose
test-docker-infrastructure-verbose: ## Run unit and integration tests with race detector and with verbose flag for docker infrastructure provider
$(MAKE) test-docker-infrastructure TEST_ARGS="$(TEST_ARGS) -v"
.PHONY: test-infrastructure-verbose
test-infrastructure-verbose: ## Run unit and integration tests with race detector and with verbose flag for docker infrastructure provider
$(MAKE) test-infrastructure TEST_ARGS="$(TEST_ARGS) -v"

.PHONY: test-docker-infrastructure-junit
test-docker-infrastructure-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests with race detector and generate a junit report for docker infrastructure provider
cd $(CAPD_DIR); set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.infra_docker.exitcode) | tee $(ARTIFACTS)/junit.infra_docker.stdout
.PHONY: test-infrastructure-junit
test-infrastructure-junit: $(SETUP_ENVTEST) $(GOTESTSUM) ## Run unit and integration tests with race detector and generate a junit report for docker infrastructure provider
cd test/infrastructure; set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit.infra_docker.exitcode) | tee $(ARTIFACTS)/junit.infra_docker.stdout
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit.infra_docker.xml --raw-command cat $(ARTIFACTS)/junit.infra_docker.stdout
exit $$(cat $(ARTIFACTS)/junit.infra_docker.exitcode)
cd $(CAPD_DIR); set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit-fuzz.infra_docker.exitcode) | tee $(ARTIFACTS)/junit-fuzz.infra_docker.stdout
cd test/infrastructure; set +o errexit; (KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -run "^TestFuzzyConversion$$" -json ./... $(TEST_ARGS); echo $$? > $(ARTIFACTS)/junit-fuzz.infra_docker.exitcode) | tee $(ARTIFACTS)/junit-fuzz.infra_docker.stdout
$(GOTESTSUM) --junitfile $(ARTIFACTS)/junit-fuzz.infra_docker.xml --raw-command cat $(ARTIFACTS)/junit-fuzz.infra_docker.stdout
exit $$(cat $(ARTIFACTS)/junit-fuzz.infra_docker.exitcode)

.PHONY: test-test-extension
test-test-extension: $(SETUP_ENVTEST) ## Run unit and integration tests for the test extension
cd $(TEST_EXTENSION_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test -race ./... $(TEST_ARGS)
Expand Down
2 changes: 2 additions & 0 deletions api/core/v1beta1/conversion.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,8 @@ func (src *ClusterClass) ConvertTo(dstRaw conversion.Hub) error {
dst.Status.Variables[i] = variable
}

dst.Spec.KubernetesVersions = restored.Spec.KubernetesVersions

return nil
}

Expand Down
1 change: 1 addition & 0 deletions api/core/v1beta1/zz_generated.conversion.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 12 additions & 0 deletions api/core/v1beta2/clusterclass_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,18 @@ type ClusterClassSpec struct {
// +kubebuilder:validation:MinItems=1
// +kubebuilder:validation:MaxItems=1000
Patches []ClusterClassPatch `json:"patches,omitempty"`

// kubernetesVersions is the list of Kubernetes versions that can be
// used for clusters using this ClusterClass.
// The list of version must be ordered from the older to the newer version, and there should be
// at least one version for every minor in between the first and the last version.
// +optional
// +listType=atomic
// +kubebuilder:validation:MinItems=1
// +kubebuilder:validation:MaxItems=100
// +kubebuilder:validation:items:MinLength=1
// +kubebuilder:validation:items:MaxLength=256
KubernetesVersions []string `json:"kubernetesVersions,omitempty"`
}

// InfrastructureClass defines the class for the infrastructure cluster.
Expand Down
4 changes: 4 additions & 0 deletions api/core/v1beta2/common_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,10 @@ const (
// to track the name of the MachineDeployment topology it represents.
ClusterTopologyMachineDeploymentNameLabel = "topology.cluster.x-k8s.io/deployment-name"

// ClusterTopologyUpgradeStepAnnotation tracks the version of the current upgrade step.
// It is only set when an upgrade is in progress, and it contains the control plane version computed by topology controller.
ClusterTopologyUpgradeStepAnnotation = "topology.internal.cluster.x-k8s.io/upgrade-step"

// ClusterTopologyHoldUpgradeSequenceAnnotation can be used to hold the entire MachineDeployment upgrade sequence.
// If the annotation is set on a MachineDeployment topology in Cluster.spec.topology.workers, the Kubernetes upgrade
// for this MachineDeployment topology and all subsequent ones is deferred.
Expand Down
5 changes: 5 additions & 0 deletions api/core/v1beta2/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 20 additions & 0 deletions api/core/v1beta2/zz_generated.openapi.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 14 additions & 0 deletions config/crd/bases/cluster.x-k8s.io_clusterclasses.yaml
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 12 additions & 2 deletions controlplane/kubeadm/internal/controllers/scale.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,8 +185,18 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con

if feature.Gates.Enabled(feature.ClusterTopology) {
// Block when we expect an upgrade to be propagated for topology clusters.
if controlPlane.Cluster.Spec.Topology.IsDefined() && controlPlane.Cluster.Spec.Topology.Version != controlPlane.KCP.Spec.Version {
logger.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated from Cluster.spec.topology", controlPlane.Cluster.Spec.Topology.Version))
// NOTE: in case the cluster is performing an upgrade, allow creation of machines for the intermediate step.
hasSameVersionOfCurrentUpgradeStep := false
if version, ok := controlPlane.Cluster.GetAnnotations()[clusterv1.ClusterTopologyUpgradeStepAnnotation]; ok {
hasSameVersionOfCurrentUpgradeStep = version == controlPlane.KCP.Spec.Version
}

if controlPlane.Cluster.Spec.Topology.IsDefined() && controlPlane.Cluster.Spec.Topology.Version != controlPlane.KCP.Spec.Version && !hasSameVersionOfCurrentUpgradeStep {
v := controlPlane.Cluster.Spec.Topology.Version
if version, ok := controlPlane.Cluster.GetAnnotations()[clusterv1.ClusterTopologyUpgradeStepAnnotation]; ok {
v = version
}
logger.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated", v))
controlPlane.PreflightCheckResults.TopologyVersionMismatch = true
return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}, nil
}
Expand Down
80 changes: 80 additions & 0 deletions controlplane/kubeadm/internal/controllers/scale_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -562,6 +562,37 @@ func TestPreflightChecks(t *testing.T) {
TopologyVersionMismatch: true,
},
},
{
name: "control plane with a pending upgrade, but not yet at the current step of the upgrade plan, should requeue",
cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
clusterv1.ClusterTopologyUpgradeStepAnnotation: "v1.32.0",
},
},
Spec: clusterv1.ClusterSpec{
Topology: clusterv1.Topology{
Version: "v1.33.0",
},
},
},
kcp: &controlplanev1.KubeadmControlPlane{
Spec: controlplanev1.KubeadmControlPlaneSpec{
Version: "v1.31.0",
},
},
machines: []*clusterv1.Machine{
{},
},

expectResult: ctrl.Result{RequeueAfter: preflightFailedRequeueAfter},
expectPreflight: internal.PreflightCheckResults{
HasDeletingMachine: false,
ControlPlaneComponentsNotHealthy: false,
EtcdClusterNotHealthy: false,
TopologyVersionMismatch: true,
},
},
{
name: "control plane with a deleting machine should requeue",
kcp: &controlplanev1.KubeadmControlPlane{},
Expand Down Expand Up @@ -687,6 +718,55 @@ func TestPreflightChecks(t *testing.T) {
TopologyVersionMismatch: false,
},
},
{
name: "control plane with a pending upgrade, but already at the current step of the upgrade plan, should pass",
cluster: &clusterv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
clusterv1.ClusterTopologyUpgradeStepAnnotation: "v1.32.0",
},
},
Spec: clusterv1.ClusterSpec{
Topology: clusterv1.Topology{
Version: "v1.33.0",
},
},
},
kcp: &controlplanev1.KubeadmControlPlane{
Spec: controlplanev1.KubeadmControlPlaneSpec{
Version: "v1.32.0",
}, Status: controlplanev1.KubeadmControlPlaneStatus{
Conditions: []metav1.Condition{
{Type: controlplanev1.KubeadmControlPlaneControlPlaneComponentsHealthyCondition, Status: metav1.ConditionTrue},
{Type: controlplanev1.KubeadmControlPlaneEtcdClusterHealthyCondition, Status: metav1.ConditionTrue},
},
},
},
machines: []*clusterv1.Machine{
{
Status: clusterv1.MachineStatus{
NodeRef: clusterv1.MachineNodeReference{
Name: "node-1",
},
Conditions: []metav1.Condition{
{Type: controlplanev1.KubeadmControlPlaneMachineAPIServerPodHealthyCondition, Status: metav1.ConditionTrue},
{Type: controlplanev1.KubeadmControlPlaneMachineControllerManagerPodHealthyCondition, Status: metav1.ConditionTrue},
{Type: controlplanev1.KubeadmControlPlaneMachineSchedulerPodHealthyCondition, Status: metav1.ConditionTrue},
{Type: controlplanev1.KubeadmControlPlaneMachineEtcdPodHealthyCondition, Status: metav1.ConditionTrue},
{Type: controlplanev1.KubeadmControlPlaneMachineEtcdMemberHealthyCondition, Status: metav1.ConditionTrue},
},
},
},
},

expectResult: ctrl.Result{},
expectPreflight: internal.PreflightCheckResults{
HasDeletingMachine: false,
ControlPlaneComponentsNotHealthy: false,
EtcdClusterNotHealthy: false,
TopologyVersionMismatch: false,
},
},
}

for _, tt := range testCases {
Expand Down
6 changes: 5 additions & 1 deletion controlplane/kubeadm/internal/controllers/status.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -836,7 +836,11 @@ func minTime(t1, t2 time.Time) time.Time {
func getPreflightMessages(cluster *clusterv1.Cluster, preflightChecks internal.PreflightCheckResults) []string {
additionalMessages := []string{}
if preflightChecks.TopologyVersionMismatch {
additionalMessages = append(additionalMessages, fmt.Sprintf("* waiting for a version upgrade to %s to be propagated from Cluster.spec.topology", cluster.Spec.Topology.Version))
v := cluster.Spec.Topology.Version
if version, ok := cluster.GetAnnotations()[clusterv1.ClusterTopologyUpgradeStepAnnotation]; ok {
v = version
}
additionalMessages = append(additionalMessages, fmt.Sprintf("* waiting for a version upgrade to %s to be propagated", v))
}

if preflightChecks.HasDeletingMachine {
Expand Down
4 changes: 2 additions & 2 deletions controlplane/kubeadm/internal/controllers/status_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,7 @@ func Test_setScalingUpCondition(t *testing.T) {
Status: metav1.ConditionTrue,
Reason: controlplanev1.KubeadmControlPlaneScalingUpReason,
Message: "Scaling up from 3 to 5 replicas is blocked because:\n" +
"* waiting for a version upgrade to v1.32.0 to be propagated from Cluster.spec.topology\n" +
"* waiting for a version upgrade to v1.32.0 to be propagated\n" +
"* waiting for a control plane Machine to complete deletion\n" +
"* waiting for control plane components to become healthy\n" +
"* waiting for etcd cluster to become healthy",
Expand Down Expand Up @@ -645,7 +645,7 @@ After above Pods have been removed from the Node, the following Pods will be evi
Status: metav1.ConditionTrue,
Reason: controlplanev1.KubeadmControlPlaneScalingDownReason,
Message: "Scaling down from 3 to 1 replicas is blocked because:\n" +
"* waiting for a version upgrade to v1.32.0 to be propagated from Cluster.spec.topology\n" +
"* waiting for a version upgrade to v1.32.0 to be propagated\n" +
"* waiting for a control plane Machine to complete deletion\n" +
"* waiting for control plane components to become healthy\n" +
"* waiting for etcd cluster to become healthy",
Expand Down
Loading