Skip to content

Commit

Permalink
Shoot conditions lastUpdateTime is no longer updated continuously
Browse files Browse the repository at this point in the history 
```improvement operator
The shoot's `.status.conditions[].lastUpdateTime` is no longer continuously updated. Note that if the gardenlet does not renew its lease anymore then the gardener-controller-manager will, after a certain threshold, set the `Shoot`'s conditions to `Unknown` to indicate that the displayed information is most likely outdated.
```
  • Loading branch information
@rfranzke
rfranzke committed Sep 28, 2020
1 parent db3c824 commit ffb2c7c
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 75 deletions.
52 changes: 22 additions & 30 deletions pkg/apis/core/v1beta1/helper/helper.go
View file
Expand Up @@ -36,33 +36,17 @@ var Now = metav1.Now

// InitCondition initializes a new Condition with an Unknown status.
func InitCondition(conditionType gardencorev1beta1.ConditionType) gardencorev1beta1.Condition {
now := Now()
return gardencorev1beta1.Condition{
Type: conditionType,
Status: gardencorev1beta1.ConditionUnknown,
Reason: "ConditionInitialized",
Message: "The condition has been initialized but its semantic check has not been performed yet.",
LastTransitionTime: Now(),
LastTransitionTime: now,
LastUpdateTime: now,
}
}

// NewConditions initializes the provided conditions based on an existing list. If a condition type does not exist
// in the list yet, it will be set to default values.
func NewConditions(conditions []gardencorev1beta1.Condition, conditionTypes ...gardencorev1beta1.ConditionType) []*gardencorev1beta1.Condition {
newConditions := []*gardencorev1beta1.Condition{}

// We retrieve the current conditions in order to update them appropriately.
for _, conditionType := range conditionTypes {
if c := GetCondition(conditions, conditionType); c != nil {
newConditions = append(newConditions, c)
continue
}
initializedCondition := InitCondition(conditionType)
newConditions = append(newConditions, &initializedCondition)
}

return newConditions
}

// GetCondition returns the condition with the given <conditionType> out of the list of <conditions>.
// In case the required type could not be found, it returns nil.
func GetCondition(conditions []gardencorev1beta1.Condition, conditionType gardencorev1beta1.ConditionType) *gardencorev1beta1.Condition {
Expand All @@ -86,19 +70,27 @@ func GetOrInitCondition(conditions []gardencorev1beta1.Condition, conditionType

// UpdatedCondition updates the properties of one specific condition.
func UpdatedCondition(condition gardencorev1beta1.Condition, status gardencorev1beta1.ConditionStatus, reason, message string, codes ...gardencorev1beta1.ErrorCode) gardencorev1beta1.Condition {
newCondition := gardencorev1beta1.Condition{
Type: condition.Type,
Status: status,
Reason: reason,
Message: message,
LastTransitionTime: condition.LastTransitionTime,
LastUpdateTime: Now(),
Codes: codes,
}
var (
newCondition = gardencorev1beta1.Condition{
Type: condition.Type,
Status: status,
Reason: reason,
Message: message,
LastTransitionTime: condition.LastTransitionTime,
LastUpdateTime: condition.LastUpdateTime,
Codes: codes,
}
now = Now()
)

if condition.Status != status {
newCondition.LastTransitionTime = Now()
newCondition.LastTransitionTime = now
}

if condition.Reason != reason || condition.Message != message || !apiequality.Semantic.DeepEqual(condition.Codes, codes) {
newCondition.LastUpdateTime = now
}

return newCondition
}

Expand Down Expand Up @@ -195,7 +187,7 @@ func ComputeOperationType(meta metav1.ObjectMeta, lastOperation *gardencorev1bet
return gardencorev1beta1.LastOperationTypeCreate
case lastOperation.Type == gardencorev1beta1.LastOperationTypeMigrate && lastOperation.State != gardencorev1beta1.LastOperationStateSucceeded:
return gardencorev1beta1.LastOperationTypeMigrate
case (lastOperation.Type == gardencorev1beta1.LastOperationTypeRestore && lastOperation.State != gardencorev1beta1.LastOperationStateSucceeded):
case lastOperation.Type == gardencorev1beta1.LastOperationTypeRestore && lastOperation.State != gardencorev1beta1.LastOperationStateSucceeded:
return gardencorev1beta1.LastOperationTypeRestore
}
return gardencorev1beta1.LastOperationTypeReconcile
Expand Down
20 changes: 19 additions & 1 deletion pkg/apis/core/v1beta1/helper/helper_test.go
View file
Expand Up @@ -62,7 +62,7 @@ var _ = Describe("helper", func() {
"Reason": Equal("reason"),
"Message": Equal("message"),
"LastTransitionTime": Equal(zeroTime),
"LastUpdateTime": Not(Equal(zeroTime)),
"LastUpdateTime": Equal(zeroTime),
}),
),
Entry("update reason",
Expand All @@ -83,6 +83,24 @@ var _ = Describe("helper", func() {
"LastUpdateTime": Not(Equal(zeroTime)),
}),
),
Entry("update message",
gardencorev1beta1.Condition{
Status: gardencorev1beta1.ConditionTrue,
Reason: "reason",
Message: "message",
},
gardencorev1beta1.ConditionTrue,
"reason",
"OtherMessage",
nil,
MatchFields(IgnoreExtras, Fields{
"Status": Equal(gardencorev1beta1.ConditionTrue),
"Reason": Equal("reason"),
"Message": Equal("OtherMessage"),
"LastTransitionTime": Equal(zeroTime),
"LastUpdateTime": Not(Equal(zeroTime)),
}),
),
Entry("update codes",
gardencorev1beta1.Condition{
Status: gardencorev1beta1.ConditionTrue,
Expand Down
95 changes: 53 additions & 42 deletions pkg/gardenlet/controller/shoot/shoot_care_control.go
View file
Expand Up @@ -199,37 +199,48 @@ func (c *defaultCareControl) Care(shootObj *gardencorev1beta1.Shoot, key string)
conditionControlPlaneHealthy = gardencorev1beta1helper.GetOrInitCondition(shoot.Status.Conditions, gardencorev1beta1.ShootControlPlaneHealthy)
conditionEveryNodeReady = gardencorev1beta1helper.GetOrInitCondition(shoot.Status.Conditions, gardencorev1beta1.ShootEveryNodeReady)
conditionSystemComponentsHealthy = gardencorev1beta1helper.GetOrInitCondition(shoot.Status.Conditions, gardencorev1beta1.ShootSystemComponentsHealthy)
oldConditions = []gardencorev1beta1.Condition{
conditionAPIServerAvailable,
conditionControlPlaneHealthy,
conditionEveryNodeReady,
conditionSystemComponentsHealthy,
}

seedConditions []gardencorev1beta1.Condition

constraintHibernationPossible = gardencorev1beta1helper.GetOrInitCondition(shoot.Status.Constraints, gardencorev1beta1.ShootHibernationPossible)
oldConstraints = []gardencorev1beta1.Condition{
constraintHibernationPossible,
}
)

botanist, err := botanistpkg.New(operation)
if err != nil {
message := fmt.Sprintf("Failed to create a botanist object to perform the care operations (%s).", err.Error())
operation.Logger.Error(message)

conditionAPIServerAvailable = gardencorev1beta1helper.UpdatedConditionUnknownErrorMessage(conditionAPIServerAvailable, message)
conditionControlPlaneHealthy = gardencorev1beta1helper.UpdatedConditionUnknownErrorMessage(conditionControlPlaneHealthy, message)
conditionEveryNodeReady = gardencorev1beta1helper.UpdatedConditionUnknownErrorMessage(conditionEveryNodeReady, message)
conditionSystemComponentsHealthy = gardencorev1beta1helper.UpdatedConditionUnknownErrorMessage(conditionSystemComponentsHealthy, message)
conditions := []gardencorev1beta1.Condition{
conditionAPIServerAvailable,
conditionControlPlaneHealthy,
conditionEveryNodeReady,
conditionSystemComponentsHealthy,
}

constraintHibernationPossible = gardencorev1beta1helper.UpdatedConditionUnknownErrorMessage(constraintHibernationPossible, message)
constraints := []gardencorev1beta1.Condition{
constraintHibernationPossible,
}

operation.Logger.Error(message)

_, _ = updateShootStatus(gardenClient.GardenCore(),
shoot,
[]gardencorev1beta1.Condition{
conditionAPIServerAvailable,
conditionControlPlaneHealthy,
conditionEveryNodeReady,
conditionSystemComponentsHealthy,
},
[]gardencorev1beta1.Condition{
constraintHibernationPossible,
},
)
if !gardencorev1beta1helper.ConditionsNeedUpdate(oldConditions, conditions) &&
!gardencorev1beta1helper.ConditionsNeedUpdate(oldConstraints, constraints) {
return nil
}

_, _ = updateShootStatus(gardenClient.GardenCore(), shoot, conditions, constraints)
return nil // We do not want to run in the exponential backoff for the condition checks.
}

Expand Down Expand Up @@ -268,60 +279,60 @@ func (c *defaultCareControl) Care(shootObj *gardencorev1beta1.Shoot, key string)
},
)(ctx)

// Update Shoot status
updatedShoot, err := updateShootStatus(gardenClient.GardenCore(),
shoot,
append(
[]gardencorev1beta1.Condition{
conditionAPIServerAvailable,
conditionControlPlaneHealthy,
conditionEveryNodeReady,
conditionSystemComponentsHealthy,
},
seedConditions...,
),
[]gardencorev1beta1.Condition{
var (
conditions = append([]gardencorev1beta1.Condition{
conditionAPIServerAvailable,
conditionControlPlaneHealthy,
conditionEveryNodeReady,
conditionSystemComponentsHealthy,
}, seedConditions...)
constraints = []gardencorev1beta1.Condition{
constraintHibernationPossible,
},
}
)
if err != nil {
botanist.Logger.Errorf("Could not update Shoot status: %+v", err)
return nil // We do not want to run in the exponential backoff for the condition checks.

// Update Shoot status if necessary
if gardencorev1beta1helper.ConditionsNeedUpdate(oldConditions, conditions) ||
gardencorev1beta1helper.ConditionsNeedUpdate(oldConstraints, constraints) {
updatedShoot, err := updateShootStatus(gardenClient.GardenCore(), shoot, conditions, constraints)
if err != nil {
botanist.Logger.Errorf("Could not update Shoot status: %+v", err)
return nil // We do not want to run in the exponential backoff for the condition checks.
}
shoot = updatedShoot
}

// Mark Shoot as healthy/unhealthy
_, err = kutil.TryUpdateShootLabels(
if _, err := kutil.TryUpdateShootLabels(
gardenClient.GardenCore(),
retry.DefaultBackoff,
updatedShoot.ObjectMeta,
shoot.ObjectMeta,
StatusLabelTransform(
ComputeStatus(
updatedShoot.Status.LastOperation,
updatedShoot.Status.LastErrors,
shoot.Status.LastOperation,
shoot.Status.LastErrors,
conditionAPIServerAvailable,
conditionControlPlaneHealthy,
conditionEveryNodeReady,
conditionSystemComponentsHealthy,
),
),
)
if err != nil {
); err != nil {
botanist.Logger.Errorf("Could not update Shoot health label: %+v", err)
return nil // We do not want to run in the exponential backoff for the condition checks.
}

return nil // We do not want to run in the exponential backoff for the condition checks.
return nil
}

func updateShootStatus(g gardencore.Interface, shoot *gardencorev1beta1.Shoot, conditions, constraints []gardencorev1beta1.Condition) (*gardencorev1beta1.Shoot, error) {
newShoot, err := kutil.TryUpdateShootStatus(g, retry.DefaultBackoff, shoot.ObjectMeta,
return kutil.TryUpdateShootStatus(g, retry.DefaultBackoff, shoot.ObjectMeta,
func(shoot *gardencorev1beta1.Shoot) (*gardencorev1beta1.Shoot, error) {
shoot.Status.Conditions = conditions
shoot.Status.Constraints = constraints
return shoot, nil
})

return newShoot, err
},
)
}

// garbageCollection cleans the Seed and the Shoot cluster from no longer required
Expand Down
4 changes: 2 additions & 2 deletions pkg/utils/kubernetes/health/health.go
View file
Expand Up @@ -341,10 +341,10 @@ func CheckAPIServerAvailability(condition gardencorev1beta1.Condition, restClien
} else {
body = string(bodyRaw)
}
message := fmt.Sprintf("API server /healthz endpoint endpoint check returned a non ok status code %d. %s (%s)", statusCode, responseDurationText, body)
message := fmt.Sprintf("API server /healthz endpoint check returned a non ok status code %d. (%s)", statusCode, body)
return conditioner("HealthzRequestError", message)
}

message := fmt.Sprintf("API server /healthz endpoint responded with success status code. %s", responseDurationText)
message := "API server /healthz endpoint responded with success status code."
return gardencorev1beta1helper.UpdatedCondition(condition, gardencorev1beta1.ConditionTrue, "HealthzRequestSucceeded", message)
}

0 comments on commit ffb2c7c

Please sign in to comment.