This repository has been archived by the owner on Nov 30, 2023. It is now read-only.
/
checkupgradeprogress.go
143 lines (119 loc) · 5.03 KB
/
checkupgradeprogress.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
package clusterreleaseversion
import (
"context"
"time"
"github.com/giantswarm/apiextensions/v6/pkg/annotation"
"github.com/giantswarm/conditions/pkg/conditions"
"github.com/giantswarm/microerror"
capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
capi "sigs.k8s.io/cluster-api/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/giantswarm/azure-operator/v7/pkg/helpers"
"github.com/giantswarm/azure-operator/v7/service/controller/key"
)
func (r *Resource) isUpgradeCompleted(ctx context.Context, cluster *capi.Cluster) (bool, error) {
r.logger.Debugf(ctx, "checking if cluster upgrade has been completed")
// Here we expect that Cluster Upgrading conditions is set to True.
upgradingCondition, upgradingConditionSet := conditions.GetUpgrading(cluster)
if !upgradingConditionSet || !conditions.IsTrue(&upgradingCondition) {
err := microerror.Maskf(
conditions.UnexpectedConditionStatusError,
conditions.ExpectedTrueErrorMessage(cluster, conditions.Upgrading))
return false, err
}
// Upgrading is in progress, now let's check if it has been completed.
// But don't check if Upgrading has been completed for the first 5 minutes,
// give other controllers time to start reconciling their CRs.
if time.Now().Before(upgradingCondition.LastTransitionTime.Add(5 * time.Minute)) {
r.logger.Debugf(ctx, "upgrade is in progress for less than 5 minutes, check back later")
return false, nil
}
controlPlaneUpgraded, err := r.isControlPlaneUpgraded(ctx, cluster)
if err != nil {
return false, microerror.Mask(err)
}
if !controlPlaneUpgraded {
r.logger.Debugf(ctx, "cluster upgrade is still in progress")
return false, nil
}
allNodePoolsUpgraded, err := r.areNodePoolsUpgraded(ctx, cluster)
if err != nil {
return false, microerror.Mask(err)
}
if !allNodePoolsUpgraded {
r.logger.Debugf(ctx, "cluster upgrade is still in progress")
return false, nil
}
r.logger.Debugf(ctx, "cluster upgrade has been completed")
return true, nil
}
func (r *Resource) isControlPlaneUpgraded(ctx context.Context, cluster *capi.Cluster) (bool, error) {
r.logger.Debugf(ctx, "checking if control plane upgrade has been completed")
azureMachineList := capz.AzureMachineList{}
err := r.ctrlClient.List(ctx, &azureMachineList, client.MatchingLabels{capi.ClusterLabelName: cluster.Name})
if err != nil {
return false, microerror.Mask(err)
}
// This is the desired cluster release version. We will check if control
// plane nodes are upgraded to this release version.
desiredClusterReleaseVersion := key.ReleaseVersion(cluster)
for i := range azureMachineList.Items {
if !isUpgradedToDesiredReleaseVersion(&azureMachineList.Items[i], desiredClusterReleaseVersion) {
r.logger.Debugf(ctx, "AzureMachine %s has not been upgraded yet", azureMachineList.Items[i].Name)
return false, nil
}
}
r.logger.Debugf(ctx, "control plane upgrade has been completed")
return true, nil
}
func (r *Resource) areNodePoolsUpgraded(ctx context.Context, cluster *capi.Cluster) (bool, error) {
r.logger.Debugf(ctx, "checking if node pools upgrade has been completed")
machinePools, err := helpers.GetMachinePoolsByMetadata(ctx, r.ctrlClient, cluster.ObjectMeta)
if err != nil {
return false, microerror.Mask(err)
}
// This is the desired cluster release version. We will check if node pools
// are upgraded to this release version.
desiredClusterReleaseVersion := key.ReleaseVersion(cluster)
for i := range machinePools.Items {
if !isUpgradedToDesiredReleaseVersion(&machinePools.Items[i], desiredClusterReleaseVersion) {
r.logger.Debugf(ctx, "MachinePool %s has not been upgraded yet", machinePools.Items[i].Name)
return false, nil
}
}
r.logger.Debugf(ctx, "node pools upgrade has been completed")
return true, nil
}
func isUpgradedToDesiredReleaseVersion(obj conditions.Object, desiredClusterReleaseVersion string) bool {
if conditions.IsCreatingTrue(obj) {
// Keeping this for backward compatibility, but it should not be necessary:
// We are in the upgrade process, but a node pool is being created. This
// is the case for first upgrade to node pools release, so an upgrade
// from v12.x.x to v13.x.x, as cluster upgrade will trigger first node
// pool creation.
return false
}
if conditions.IsUpgradingTrue(obj) {
// Control plane node or node pool is being upgraded.
return false
}
currentDesiredReleaseVersion := key.ReleaseVersion(obj)
if currentDesiredReleaseVersion != desiredClusterReleaseVersion {
// A control plane node or node pool upgrade has not been started yet.
return false
}
lastDeployedReleaseVersion, ok := obj.GetAnnotations()[annotation.LastDeployedReleaseVersion]
if !ok {
// Control plane or a node pool is still not created. This should be
// caught above in Creating check, but let's err on the side of caution
// here.
return false
}
if lastDeployedReleaseVersion != desiredClusterReleaseVersion {
// A control plane node or node pool has not yet been upgraded to the
// desired release version.
return false
}
// Upgrade completed for control plane node or node pool!
return true
}