Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
v1.113.0
v1.113.1
46 changes: 23 additions & 23 deletions pkg/gardenlet/controller/shoot/care/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"sync"
"time"

"github.com/go-logr/logr"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/utils/clock"
Expand Down Expand Up @@ -118,7 +119,8 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
shoot,
)
if err != nil {
if err := r.patchStatusToUnknown(ctx, shoot, "Precondition failed: operation could not be initialized", shootConditions.ConvertToSlice(), shootConstraints.ConvertToSlice()); err != nil {
updatedConditions, updatedConstraints := r.setStatusToUnknown("Precondition failed: operation could not be initialized", shootConditions.ConvertToSlice(), shootConstraints.ConvertToSlice())
if err := r.patchStatus(ctx, log, shoot, shootConditions, updatedConditions, shootConstraints, updatedConstraints); err != nil {
log.Error(err, "Error when trying to update the shoot status after failed operation initialization")
}
return reconcile.Result{}, err
Expand Down Expand Up @@ -182,19 +184,9 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
return reconcile.Result{}, err
}

// Update Shoot status (conditions, constraints) if necessary
if v1beta1helper.ConditionsNeedUpdate(shootConditions.ConvertToSlice(), updatedConditions) ||
v1beta1helper.ConditionsNeedUpdate(shootConstraints.ConvertToSlice(), updatedConstraints) {
log.V(1).Info("Updating status conditions and constraints")
// Rebuild shoot conditions and constraints to ensure that only the conditions and constraints with the
// correct types will be updated, and any other conditions will remain intact
conditions := v1beta1helper.BuildConditions(shoot.Status.Conditions, updatedConditions, shootConditions.ConditionTypes())
constraints := v1beta1helper.BuildConditions(shoot.Status.Constraints, updatedConstraints, shootConstraints.ConstraintTypes())

if err := r.patchStatus(ctx, shoot, conditions, constraints); err != nil {
log.Error(err, "Error when trying to update the shoot status")
return reconcile.Result{}, err
}
if err := r.patchStatus(ctx, log, shoot, shootConditions, updatedConditions, shootConstraints, updatedConstraints); err != nil {
log.Error(err, "Error when trying to update the shoot status")
return reconcile.Result{}, err
}

return reconcile.Result{RequeueAfter: r.Config.Controllers.ShootCare.SyncPeriod.Duration}, nil
Expand All @@ -208,14 +200,26 @@ func (r *Reconciler) conditionThresholdsToProgressingMapping() map[gardencorev1b
return out
}

func (r *Reconciler) patchStatus(ctx context.Context, shoot *gardencorev1beta1.Shoot, conditions, constraints []gardencorev1beta1.Condition) error {
func (r *Reconciler) patchStatus(ctx context.Context, log logr.Logger, shoot *gardencorev1beta1.Shoot, existingConditions ShootConditions, updatedConditions []gardencorev1beta1.Condition, existingConstraints ShootConstraints, updatedConstraints []gardencorev1beta1.Condition) error {
// Update Shoot status (conditions, constraints) only if necessary
if !v1beta1helper.ConditionsNeedUpdate(existingConditions.ConvertToSlice(), updatedConditions) && !v1beta1helper.ConditionsNeedUpdate(existingConstraints.ConvertToSlice(), updatedConstraints) {
return nil
}

// Rebuild shoot conditions and constraints to ensure that only the conditions and constraints with the
// correct types will be updated, and any other conditions will remain intact
mergedConditions := v1beta1helper.BuildConditions(shoot.Status.Conditions, updatedConditions, existingConditions.ConditionTypes())
mergedConstraints := v1beta1helper.BuildConditions(shoot.Status.Constraints, updatedConstraints, existingConstraints.ConstraintTypes())

log.V(1).Info("Updating status conditions and constraints")

patch := client.StrategicMergeFrom(shoot.DeepCopy())
shoot.Status.Conditions = conditions
shoot.Status.Constraints = constraints
shoot.Status.Conditions = mergedConditions
shoot.Status.Constraints = mergedConstraints
return r.GardenClient.Status().Patch(ctx, shoot, patch)
}

func (r *Reconciler) patchStatusToUnknown(ctx context.Context, shoot *gardencorev1beta1.Shoot, message string, conditions, constraints []gardencorev1beta1.Condition) error {
func (r *Reconciler) setStatusToUnknown(message string, conditions []gardencorev1beta1.Condition, constraints []gardencorev1beta1.Condition) ([]gardencorev1beta1.Condition, []gardencorev1beta1.Condition) {
updatedConditions := make([]gardencorev1beta1.Condition, 0, len(conditions))
for _, cond := range conditions {
updatedConditions = append(updatedConditions, v1beta1helper.UpdatedConditionUnknownErrorMessageWithClock(r.Clock, cond, message))
Expand All @@ -226,11 +230,7 @@ func (r *Reconciler) patchStatusToUnknown(ctx context.Context, shoot *gardencore
updatedConstraints = append(updatedConstraints, v1beta1helper.UpdatedConditionUnknownErrorMessageWithClock(r.Clock, constr, message))
}

if !v1beta1helper.ConditionsNeedUpdate(conditions, updatedConditions) && !v1beta1helper.ConditionsNeedUpdate(constraints, updatedConstraints) {
return nil
}

return r.patchStatus(ctx, shoot, updatedConditions, updatedConstraints)
return updatedConditions, updatedConstraints
}

func shootClientInitializer(ctx context.Context, o *operation.Operation) func() (kubernetes.Interface, bool, error) {
Expand Down
103 changes: 73 additions & 30 deletions pkg/gardenlet/controller/shoot/care/reconciler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/go-logr/logr"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/onsi/gomega/gstruct"
"github.com/onsi/gomega/types"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -125,10 +124,28 @@ var _ = Describe("Shoot Care Control", func() {

Context("when health check setup is broken", func() {
Context("when operation cannot be created", func() {
extraneousCondition := gardencorev1beta1.Condition{
Type: "foo",
Status: gardencorev1beta1.ConditionTrue,
Reason: "test",
Message: "test",
}

extraneousConstraint := gardencorev1beta1.Condition{
Type: "bar",
Status: gardencorev1beta1.ConditionTrue,
Reason: "test",
Message: "test",
}

JustBeforeEach(func() {
fakeErr := errors.New("foo")
DeferCleanup(test.WithVar(&NewOperation, opFunc(nil, fakeErr)))

shoot.Status.Conditions = append(shoot.Status.Conditions, extraneousCondition)
shoot.Status.Constraints = append(shoot.Status.Constraints, extraneousConstraint)
Expect(gardenClient.Status().Update(ctx, shoot)).To(Succeed())

reconciler = &Reconciler{
GardenClient: gardenClient,
SeedClientSet: kubernetesfake.NewClientSet(),
Expand All @@ -146,8 +163,20 @@ var _ = Describe("Shoot Care Control", func() {
It("should report a setup failure", func() {
updatedShoot := &gardencorev1beta1.Shoot{}
Expect(gardenClient.Get(ctx, client.ObjectKeyFromObject(shoot), updatedShoot)).To(Succeed())
Expect(updatedShoot.Status.Conditions).To(consistOfConditionsInUnknownStatus("Precondition failed: operation could not be initialized", v1beta1helper.IsWorkerless(shoot)))
Expect(updatedShoot.Status.Constraints).To(consistOfConstraintsInUnknownStatus("Precondition failed: operation could not be initialized"))
Expect(updatedShoot.Status.Conditions).To(containConditionsInUnknownStatus("Precondition failed: operation could not be initialized", v1beta1helper.IsWorkerless(shoot)))
Expect(updatedShoot.Status.Constraints).To(containConstraintsInUnknownStatus("Precondition failed: operation could not be initialized"))
Expect(updatedShoot.Status.Conditions).To(ContainCondition(
OfType(extraneousCondition.Type),
WithStatus(extraneousCondition.Status),
WithReason(extraneousCondition.Reason),
WithMessage(extraneousCondition.Message),
))
Expect(updatedShoot.Status.Constraints).To(ContainCondition(
OfType(extraneousConstraint.Type),
WithStatus(extraneousConstraint.Status),
WithReason(extraneousConstraint.Reason),
WithMessage(extraneousConstraint.Message),
))
})
})

Expand All @@ -159,8 +188,20 @@ var _ = Describe("Shoot Care Control", func() {
It("should report a setup failure", func() {
updatedShoot := &gardencorev1beta1.Shoot{}
Expect(gardenClient.Get(ctx, client.ObjectKeyFromObject(shoot), updatedShoot)).To(Succeed())
Expect(updatedShoot.Status.Conditions).To(consistOfConditionsInUnknownStatus("Precondition failed: operation could not be initialized", v1beta1helper.IsWorkerless(shoot)))
Expect(updatedShoot.Status.Constraints).To(consistOfConstraintsInUnknownStatus("Precondition failed: operation could not be initialized"))
Expect(updatedShoot.Status.Conditions).To(containConditionsInUnknownStatus("Precondition failed: operation could not be initialized", v1beta1helper.IsWorkerless(shoot)))
Expect(updatedShoot.Status.Constraints).To(containConstraintsInUnknownStatus("Precondition failed: operation could not be initialized"))
Expect(updatedShoot.Status.Conditions).To(ContainCondition(
OfType(extraneousCondition.Type),
WithStatus(extraneousCondition.Status),
WithReason(extraneousCondition.Reason),
WithMessage(extraneousCondition.Message),
))
Expect(updatedShoot.Status.Constraints).To(ContainCondition(
OfType(extraneousConstraint.Type),
WithStatus(extraneousConstraint.Status),
WithReason(extraneousConstraint.Reason),
WithMessage(extraneousConstraint.Message),
))
})
})
})
Expand Down Expand Up @@ -568,8 +609,8 @@ func nopGarbageCollectorFunc() NewGarbageCollectorFunc {
}
}

func consistOfConditionsInUnknownStatus(message string, isWorkerless bool) types.GomegaMatcher {
var expectedLength = 4
func containConditionsInUnknownStatus(message string, isWorkerless bool) types.GomegaMatcher {
var expectedLength = 5
matcher := And(
ContainCondition(
OfType(gardencorev1beta1.ShootAPIServerAvailable),
Expand All @@ -593,7 +634,7 @@ func consistOfConditionsInUnknownStatus(message string, isWorkerless bool) types
)

if !isWorkerless {
expectedLength = 5
expectedLength = 6
matcher = And(matcher,
ContainCondition(
OfType(gardencorev1beta1.ShootEveryNodeReady),
Expand All @@ -606,27 +647,29 @@ func consistOfConditionsInUnknownStatus(message string, isWorkerless bool) types
return And(matcher, HaveLen(expectedLength))
}

func consistOfConstraintsInUnknownStatus(message string) types.GomegaMatcher {
return ConsistOf(
MatchFields(IgnoreExtras, Fields{
"Type": Equal(gardencorev1beta1.ShootHibernationPossible),
"Status": Equal(gardencorev1beta1.ConditionUnknown),
"Message": Equal(message),
}),
MatchFields(IgnoreExtras, Fields{
"Type": Equal(gardencorev1beta1.ShootMaintenancePreconditionsSatisfied),
"Status": Equal(gardencorev1beta1.ConditionUnknown),
"Message": Equal(message),
}),
MatchFields(IgnoreExtras, Fields{
"Type": Equal(gardencorev1beta1.ShootCACertificateValiditiesAcceptable),
"Status": Equal(gardencorev1beta1.ConditionUnknown),
"Message": Equal(message),
}),
MatchFields(IgnoreExtras, Fields{
"Type": Equal(gardencorev1beta1.ShootCRDsWithProblematicConversionWebhooks),
"Status": Equal(gardencorev1beta1.ConditionUnknown),
"Message": Equal(message),
}),
func containConstraintsInUnknownStatus(message string) types.GomegaMatcher {
var expectedLength = 5
matcher := And(
ContainCondition(
OfType(gardencorev1beta1.ShootHibernationPossible),
WithStatus(gardencorev1beta1.ConditionUnknown),
WithMessage(message),
),
ContainCondition(
OfType(gardencorev1beta1.ShootMaintenancePreconditionsSatisfied),
WithStatus(gardencorev1beta1.ConditionUnknown),
WithMessage(message),
),
ContainCondition(
OfType(gardencorev1beta1.ShootCACertificateValiditiesAcceptable),
WithStatus(gardencorev1beta1.ConditionUnknown),
WithMessage(message),
), ContainCondition(
OfType(gardencorev1beta1.ShootCRDsWithProblematicConversionWebhooks),
WithStatus(gardencorev1beta1.ConditionUnknown),
WithMessage(message),
),
)

return And(matcher, HaveLen(expectedLength))
}
Original file line number Diff line number Diff line change
Expand Up @@ -106,22 +106,26 @@ func (r *Reconciler) runForceDeleteShootFlow(ctx context.Context, log logr.Logge
Fn: flow.TaskFn(func(ctx context.Context) error {
return botanist.Shoot.Components.ControlPlane.MachineControllerManager.Destroy(ctx)
}),
SkipIf: botanist.Shoot.IsWorkerless,
})
waitUntilMachineControllerManagerDeleted = g.Add(flow.Task{
Name: "Waiting until machine-controller-manager has been deleted",
Fn: flow.TaskFn(func(ctx context.Context) error {
return botanist.Shoot.Components.ControlPlane.MachineControllerManager.WaitCleanup(ctx)
}),
SkipIf: botanist.Shoot.IsWorkerless,
Dependencies: flow.NewTaskIDs(deleteMachineControllerManager),
})
deleteMachineResources = g.Add(flow.Task{
Name: "Deleting machine resources",
Fn: flow.TaskFn(cleaner.DeleteMachineResources).RetryUntilTimeout(defaultInterval, defaultTimeout),
SkipIf: botanist.Shoot.IsWorkerless,
Dependencies: flow.NewTaskIDs(waitUntilMachineControllerManagerDeleted),
})
waitUntilMachineResourcesDeleted = g.Add(flow.Task{
Name: "Waiting until machine resources have been deleted",
Fn: flow.TaskFn(cleaner.WaitUntilMachineResourcesDeleted).Timeout(defaultTimeout),
SkipIf: botanist.Shoot.IsWorkerless,
Dependencies: flow.NewTaskIDs(deleteMachineResources),
})
setKeepObjectsForManagedResources = g.Add(flow.Task{
Expand Down
13 changes: 7 additions & 6 deletions pkg/gardenlet/controller/shoot/shoot/reconciler_reconcile.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope
kubeProxyEnabled = v1beta1helper.KubeProxyEnabled(o.Shoot.GetInfo().Spec.Kubernetes.KubeProxy)
deployKubeAPIServerTaskTimeout = defaultTimeout
shootSSHAccessEnabled = v1beta1helper.ShootEnablesSSHAccess(o.Shoot.GetInfo())
isRestoringHAControlPlane = botanist.IsRestorePhase() && v1beta1helper.IsHAControlPlaneConfigured(o.Shoot.GetInfo())
)

// During the 'Preparing' phase of different rotation operations, components are deployed twice. Also, the
Expand Down Expand Up @@ -296,7 +297,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope
waitUntilEtcdReady = g.Add(flow.Task{
Name: "Waiting until main and event etcd report readiness",
Fn: botanist.WaitUntilEtcdsReady,
SkipIf: o.Shoot.HibernationEnabled || skipReadiness,
SkipIf: (!isRestoringHAControlPlane && o.Shoot.HibernationEnabled) || skipReadiness,
Dependencies: flow.NewTaskIDs(deployETCD),
})
deployExtensionResourcesBeforeKAPI = g.Add(flow.Task{
Expand Down Expand Up @@ -333,13 +334,13 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope
scaleEtcdAfterRestore = g.Add(flow.Task{
Name: "Scaling main and events etcd after kube-apiserver is ready",
Fn: flow.TaskFn(botanist.ScaleUpETCD).RetryUntilTimeout(defaultInterval, helper.GetEtcdDeployTimeout(o.Shoot, defaultTimeout)),
SkipIf: !v1beta1helper.IsHAControlPlaneConfigured(botanist.Shoot.GetInfo()) || !botanist.IsRestorePhase() || o.Shoot.HibernationEnabled || skipReadiness,
Dependencies: flow.NewTaskIDs(waitUntilKubeAPIServerIsReady),
SkipIf: !isRestoringHAControlPlane,
Dependencies: flow.NewTaskIDs(waitUntilEtcdReady, waitUntilKubeAPIServerIsReady),
})
_ = g.Add(flow.Task{
waitUntilEtcdScaledAfterRestore = g.Add(flow.Task{
Name: "Waiting until main and events etcd scaled up after kube-apiserver is ready",
Fn: flow.TaskFn(botanist.WaitUntilEtcdsReady),
SkipIf: !v1beta1helper.IsHAControlPlaneConfigured(botanist.Shoot.GetInfo()) || !botanist.IsRestorePhase() || o.Shoot.HibernationEnabled || skipReadiness,
SkipIf: !isRestoringHAControlPlane || skipReadiness,
Dependencies: flow.NewTaskIDs(scaleEtcdAfterRestore),
})
deployGardenerResourceManager = g.Add(flow.Task{
Expand Down Expand Up @@ -896,7 +897,7 @@ func (r *Reconciler) runReconcileShootFlow(ctx context.Context, o *operation.Ope
Name: "Hibernating control plane",
Fn: flow.TaskFn(botanist.HibernateControlPlane).RetryUntilTimeout(defaultInterval, 2*time.Minute),
SkipIf: !o.Shoot.HibernationEnabled,
Dependencies: flow.NewTaskIDs(initializeShootClients, deployPrometheus, deployAlertmanager, deploySeedLogging, deployClusterAutoscaler, waitUntilWorkerReady, waitUntilExtensionResourcesAfterKAPIReady),
Dependencies: flow.NewTaskIDs(initializeShootClients, deployPrometheus, deployAlertmanager, deploySeedLogging, deployClusterAutoscaler, waitUntilWorkerReady, waitUntilExtensionResourcesAfterKAPIReady, waitUntilEtcdScaledAfterRestore),
})

// logic is inverted here
Expand Down