Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

storage-version: update conditions #96825

Merged
merged 2 commits into from Dec 14, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions pkg/controller/storageversiongc/BUILD
Expand Up @@ -14,6 +14,7 @@ go_library(
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/apiserver/pkg/storageversion:go_default_library",
"//staging/src/k8s.io/client-go/informers/apiserverinternal/v1alpha1:go_default_library",
"//staging/src/k8s.io/client-go/informers/coordination/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
Expand Down
22 changes: 2 additions & 20 deletions pkg/controller/storageversiongc/gc_controller.go
Expand Up @@ -28,6 +28,7 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/storageversion"
apiserverinternalinformers "k8s.io/client-go/informers/apiserverinternal/v1alpha1"
coordinformers "k8s.io/client-go/informers/coordination/v1"
"k8s.io/client-go/kubernetes"
Expand Down Expand Up @@ -265,32 +266,13 @@ func (c *Controller) enqueueLease(obj *coordinationv1.Lease) {
c.leaseQueue.Add(obj.Name)
}

func setCommonEncodingVersion(sv *apiserverinternalv1alpha1.StorageVersion) {
if len(sv.Status.StorageVersions) == 0 {
return
}
firstVersion := sv.Status.StorageVersions[0].EncodingVersion
agreed := true
for _, ssv := range sv.Status.StorageVersions {
if ssv.EncodingVersion != firstVersion {
agreed = false
break
}
}
if agreed {
sv.Status.CommonEncodingVersion = &firstVersion
} else {
sv.Status.CommonEncodingVersion = nil
}
}

func (c *Controller) updateOrDeleteStorageVersion(sv *apiserverinternalv1alpha1.StorageVersion, serverStorageVersions []apiserverinternalv1alpha1.ServerStorageVersion) error {
if len(serverStorageVersions) == 0 {
return c.kubeclientset.InternalV1alpha1().StorageVersions().Delete(
context.TODO(), sv.Name, metav1.DeleteOptions{})
}
sv.Status.StorageVersions = serverStorageVersions
setCommonEncodingVersion(sv)
storageversion.SetCommonEncodingVersion(sv)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is updateOrDeleteStorageVersion operating on an object obtained from an informer or a live lookup version?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a live lookup version

_, err := c.kubeclientset.InternalV1alpha1().StorageVersions().UpdateStatus(
context.TODO(), sv, metav1.UpdateOptions{})
return err
Expand Down
1 change: 1 addition & 0 deletions staging/src/k8s.io/apiserver/pkg/storageversion/BUILD
Expand Up @@ -32,6 +32,7 @@ go_test(
embed = [":go_default_library"],
deps = [
"//staging/src/k8s.io/api/apiserverinternal/v1alpha1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/github.com/google/go-cmp/cmp:go_default_library",
],
Expand Down
95 changes: 81 additions & 14 deletions staging/src/k8s.io/apiserver/pkg/storageversion/updater.go
Expand Up @@ -35,23 +35,90 @@ type Client interface {
Get(context.Context, string, metav1.GetOptions) (*v1alpha1.StorageVersion, error)
}

func setCommonEncodingVersion(sv *v1alpha1.StorageVersion) {
if len(sv.Status.StorageVersions) == 0 {
return
// SetCommonEncodingVersion updates the CommonEncodingVersion and the AllEncodingVersionsEqual
// condition based on the StorageVersions.
func SetCommonEncodingVersion(sv *v1alpha1.StorageVersion) {
var oldCommonEncodingVersion *string
if sv.Status.CommonEncodingVersion != nil {
version := *sv.Status.CommonEncodingVersion
oldCommonEncodingVersion = &version
}
firstVersion := sv.Status.StorageVersions[0].EncodingVersion
agreed := true
for _, ssv := range sv.Status.StorageVersions {
if ssv.EncodingVersion != firstVersion {
agreed = false
break
sv.Status.CommonEncodingVersion = nil
if len(sv.Status.StorageVersions) != 0 {
firstVersion := sv.Status.StorageVersions[0].EncodingVersion
agreed := true
for _, ssv := range sv.Status.StorageVersions {
if ssv.EncodingVersion != firstVersion {
agreed = false
break
}
}
if agreed {
sv.Status.CommonEncodingVersion = &firstVersion
}
}

condition := v1alpha1.StorageVersionCondition{
Type: v1alpha1.AllEncodingVersionsEqual,
Status: v1alpha1.ConditionFalse,
ObservedGeneration: sv.Generation,
LastTransitionTime: metav1.NewTime(time.Now()),
Reason: "CommonEncodingVersionUnset",
Message: "Common encoding version unset",
Comment on lines +66 to +67
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

for free-form text we are likely to tweak in future versions, make sure we won't eternally duel with a future API server that sets a slightly different version of this condition. xref things like #78133

}
if sv.Status.CommonEncodingVersion != nil {
condition.Status = v1alpha1.ConditionTrue
condition.Reason = "CommonEncodingVersionSet"
condition.Message = "Common encoding version set"
}
forceTransition := false
if oldCommonEncodingVersion != nil && sv.Status.CommonEncodingVersion != nil &&
*oldCommonEncodingVersion != *sv.Status.CommonEncodingVersion {
forceTransition = true
}
setStatusCondition(&sv.Status.Conditions, condition, forceTransition)
}

func findStatusCondition(conditions []v1alpha1.StorageVersionCondition,
conditionType v1alpha1.StorageVersionConditionType) *v1alpha1.StorageVersionCondition {
for i := range conditions {
if conditions[i].Type == conditionType {
return &conditions[i]
}
}
if agreed {
sv.Status.CommonEncodingVersion = &firstVersion
} else {
sv.Status.CommonEncodingVersion = nil
return nil
}

// setStatusCondition sets the corresponding condition in conditions to newCondition.
// conditions must be non-nil.
// 1. if the condition of the specified type already exists: all fields of the existing condition are updated to
// newCondition, LastTransitionTime is set to now if the new status differs from the old status
// 2. if a condition of the specified type does not exist: LastTransitionTime is set to now() if unset,
// and newCondition is appended
// NOTE: forceTransition allows overwriting LastTransitionTime even when the status doesn't change.
func setStatusCondition(conditions *[]v1alpha1.StorageVersionCondition, newCondition v1alpha1.StorageVersionCondition,
forceTransition bool) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How about versionChanged?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'd like to keep the current name so that the function can be used for other storage version conditions in future

if conditions == nil {
return
}

if newCondition.LastTransitionTime.IsZero() {
newCondition.LastTransitionTime = metav1.NewTime(time.Now())
}
existingCondition := findStatusCondition(*conditions, newCondition.Type)
if existingCondition == nil {
*conditions = append(*conditions, newCondition)
return
}

statusChanged := existingCondition.Status != newCondition.Status
if statusChanged || forceTransition {
existingCondition.LastTransitionTime = newCondition.LastTransitionTime
}
existingCondition.Status = newCondition.Status
existingCondition.Reason = newCondition.Reason
existingCondition.Message = newCondition.Message
Comment on lines +119 to +120
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is the bit that seems likely to duel... should we skip if the ObservedGeneration of the existing condition is already set to the current generation?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

StorageVersion has no spec, so the ObservedGeneration is always 0.

IIUC, #78133 happened because one controller's update would always trigger another controller to do calculation and update. This doesn't apply to storage version because:

  • The GC only tries updating status if the live lookup object has stale record. The watch event of one controller finishing GC won't trigger another controller to do GC again.
  • The StorageVersion manager doesn't watch.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The two controllers updating the condition are the storageVersionGC and the API server. One storageVersion GC or on API server updating the condition of a storageVersion won't cause other GC or API server update the storageVersion again, so this won't cause a duel.

existingCondition.ObservedGeneration = newCondition.ObservedGeneration
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I simplified the function a little, and it still passed the unit test, how about:

 func setStatusCondition(conditions *[]v1alpha1.StorageVersionCondition, newCondition v1alpha1.StorageVersionCondition,
     versionChanged bool) {
     if conditions == nil {
         return
     }
     existingCondition := findStatusCondition(*conditions, newCondition.Type)
     if existingCondition == nil {
         *conditions = append(*conditions, newCondition)
         return
     }
     statusChanged := existingCondition.Status != newCondition.Status
     if statusChanged || versionChanged {
         existingCondition.LastTransitionTime = newCondition.LastTransitionTime
     }
     existingCondition.Status = newCondition.Status
     existingCondition.Reason = newCondition.Reason
     existingCondition.Message = newCondition.Message
     existingCondition.ObservedGeneration = newCondition.ObservedGeneration
 }

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it. Then how about

 func setStatusCondition(conditions *[]v1alpha1.StorageVersionCondition, newCondition v1alpha1.StorageVersionCondition,
     forceTransition bool) {
     if conditions == nil {
         return
     }
     if newCondition.LastTransitionTime.IsZero() {
          newCondition.LastTransitionTime = metav1.NewTime(time.Now())
     }
     existingCondition := findStatusCondition(*conditions, newCondition.Type)
     if existingCondition == nil {
         *conditions = append(*conditions, newCondition)
         return
     }
     statusChanged := existingCondition.Status != newCondition.Status
     if statusChanged || forceTransition {
         existingCondition.LastTransitionTime = newCondition.LastTransitionTime
     }
     existingCondition.Status = newCondition.Status
     existingCondition.Reason = newCondition.Reason
     existingCondition.Message = newCondition.Message
     existingCondition.ObservedGeneration = newCondition.ObservedGeneration
 }

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

updated


// updateStorageVersionFor updates the storage version object for the resource.
Expand Down Expand Up @@ -123,6 +190,6 @@ func localUpdateStorageVersion(sv *v1alpha1.StorageVersion, apiserverID, encodin
if !foundSSV {
sv.Status.StorageVersions = append(sv.Status.StorageVersions, newSSV)
}
setCommonEncodingVersion(sv)
SetCommonEncodingVersion(sv)
return sv
}