Merge pull request #63830 from mbohlool/crd_versioning_nop

Automatic merge from submit-queue. If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

Crd versioning with nop Conversion

Implements Custom Resource Definition versioning according to[ design doc](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/api-machinery/customresources-versioning.md).

Note: I recreated this PR instead of #63518. Huge number of comments there broke github. 

@sttts @nikhita @deads2k @liggitt @lavalamp 

```release-note
Add CRD Versioning with NOP converter
```

pkg/master/controller/crdregistration/crdregistration_controller.go

@@ -77,9 +77,11 @@ func NewAutoRegistrationController(crdinformer crdinformers.CustomResourceDefini
 			cast := obj.(*apiextensions.CustomResourceDefinition)
 			c.enqueueCRD(cast)
 		},
-		UpdateFunc: func(_, obj interface{}) {
-			cast := obj.(*apiextensions.CustomResourceDefinition)
-			c.enqueueCRD(cast)
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			// Enqueue both old and new object to make sure we remove and add appropriate API services.
+			// The working queue will resolve any duplicates and only changes will stay in the queue.
+			c.enqueueCRD(oldObj.(*apiextensions.CustomResourceDefinition))
+			c.enqueueCRD(newObj.(*apiextensions.CustomResourceDefinition))
 		},
 		DeleteFunc: func(obj interface{}) {
 			cast, ok := obj.(*apiextensions.CustomResourceDefinition)
@@ -120,8 +122,10 @@ func (c *crdRegistrationController) Run(threadiness int, stopCh <-chan struct{})
 		utilruntime.HandleError(err)
 	} else {
 		for _, crd := range crds {
-			if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version}); err != nil {
-				utilruntime.HandleError(err)
+			for _, version := range crd.Spec.Versions {
+				if err := c.syncHandler(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name}); err != nil {
+					utilruntime.HandleError(err)
+				}
 			}
 		}
 	}
@@ -182,11 +186,12 @@ func (c *crdRegistrationController) processNextWorkItem() bool {
 }
 
 func (c *crdRegistrationController) enqueueCRD(crd *apiextensions.CustomResourceDefinition) {
-	c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: crd.Spec.Version})
+	for _, version := range crd.Spec.Versions {
+		c.queue.Add(schema.GroupVersion{Group: crd.Spec.Group, Version: version.Name})
+	}
 }
 
 func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.GroupVersion) error {
-	found := false
 	apiServiceName := groupVersion.Version + "." + groupVersion.Group
 
 	// check all CRDs.  There shouldn't that many, but if we have problems later we can index them
@@ -195,26 +200,27 @@ func (c *crdRegistrationController) handleVersionUpdate(groupVersion schema.Grou
 		return err
 	}
 	for _, crd := range crds {
-		if crd.Spec.Version == groupVersion.Version && crd.Spec.Group == groupVersion.Group {
-			found = true
-			break
+		if crd.Spec.Group != groupVersion.Group {
+			continue
 		}
-	}
+		for _, version := range crd.Spec.Versions {
+			if version.Name != groupVersion.Version || !version.Served {
+				continue
+			}
 
-	if !found {
-		c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName)
-		return nil
+			c.apiServiceRegistration.AddAPIServiceToSync(&apiregistration.APIService{
+				ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
+				Spec: apiregistration.APIServiceSpec{
+					Group:                groupVersion.Group,
+					Version:              groupVersion.Version,
+					GroupPriorityMinimum: 1000, // CRDs should have relatively low priority
+					VersionPriority:      100,  // CRDs will be sorted by kube-like versions like any other APIService with the same VersionPriority
+				},
+			})
+			return nil
+		}
 	}
 
-	c.apiServiceRegistration.AddAPIServiceToSync(&apiregistration.APIService{
-		ObjectMeta: metav1.ObjectMeta{Name: apiServiceName},
-		Spec: apiregistration.APIServiceSpec{
-			Group:                groupVersion.Group,
-			Version:              groupVersion.Version,
-			GroupPriorityMinimum: 1000, // CRDs should have relatively low priority
-			VersionPriority:      100,  // CRDs should have relatively low priority
-		},
-	})
-
+	c.apiServiceRegistration.RemoveAPIServiceToSync(apiServiceName)
 	return nil
 }

pkg/master/controller/crdregistration/crdregistration_controller_test.go

@@ -42,8 +42,16 @@ func TestHandleVersionUpdate(t *testing.T) {
 			startingCRDs: []*apiextensions.CustomResourceDefinition{
 				{
 					Spec: apiextensions.CustomResourceDefinitionSpec{
-						Group:   "group.com",
-						Version: "v1",
+						Group: "group.com",
+						// Version field is deprecated and crd registration won't rely on it at all.
+						// defaulting route will fill up Versions field if user only provided version field.
+						Versions: []apiextensions.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Served:  true,
+								Storage: true,
+							},
+						},
 					},
 				},
 			},
@@ -66,8 +74,14 @@ func TestHandleVersionUpdate(t *testing.T) {
 			startingCRDs: []*apiextensions.CustomResourceDefinition{
 				{
 					Spec: apiextensions.CustomResourceDefinitionSpec{
-						Group:   "group.com",
-						Version: "v1",
+						Group: "group.com",
+						Versions: []apiextensions.CustomResourceDefinitionVersion{
+							{
+								Name:    "v1",
+								Served:  true,
+								Storage: true,
+							},
+						},
 					},
 				},
 			},
@@ -98,7 +112,6 @@ func TestHandleVersionUpdate(t *testing.T) {
 			t.Errorf("%s expected %v, got %v", test.name, test.expectedRemoved, registration.removed)
 		}
 	}
-
 }
 
 type fakeAPIServiceRegistration struct {

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/fuzzer/fuzzer.go

@@ -42,6 +42,28 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 			if len(obj.Names.ListKind) == 0 && len(obj.Names.Kind) > 0 {
 				obj.Names.ListKind = obj.Names.Kind + "List"
 			}
+			if len(obj.Versions) == 0 && len(obj.Version) != 0 {
+				obj.Versions = []apiextensions.CustomResourceDefinitionVersion{
+					{
+						Name:    obj.Version,
+						Served:  true,
+						Storage: true,
+					},
+				}
+			} else if len(obj.Versions) != 0 {
+				obj.Version = obj.Versions[0].Name
+			}
+		},
+		func(obj *apiextensions.CustomResourceDefinition, c fuzz.Continue) {
+			c.FuzzNoCustom(obj)
+
+			if len(obj.Status.StoredVersions) == 0 {
+				for _, v := range obj.Spec.Versions {
+					if v.Storage && !apiextensions.IsStoredVersion(obj, v.Name) {
+						obj.Status.StoredVersions = append(obj.Status.StoredVersions, v.Name)
+					}
+				}
+			}
 		},
 		func(obj *apiextensions.JSONSchemaProps, c fuzz.Continue) {
 			// we cannot use c.FuzzNoCustom because of the interface{} fields. So let's loop with reflection.

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/helpers.go

@@ -17,6 +17,7 @@ limitations under the License.
 package apiextensions
 
 import (
+	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -116,3 +117,33 @@ func CRDRemoveFinalizer(crd *CustomResourceDefinition, needle string) {
 	}
 	crd.Finalizers = newFinalizers
 }
+
+// HasServedCRDVersion returns true if `version` is in the list of CRD's versions and the Served flag is set.
+func HasServedCRDVersion(crd *CustomResourceDefinition, version string) bool {
+	for _, v := range crd.Spec.Versions {
+		if v.Name == version {
+			return v.Served
+		}
+	}
+	return false
+}
+
+// GetCRDStorageVersion returns the storage version for given CRD.
+func GetCRDStorageVersion(crd *CustomResourceDefinition) (string, error) {
+	for _, v := range crd.Spec.Versions {
+		if v.Storage {
+			return v.Name, nil
+		}
+	}
+	// This should not happened if crd is valid
+	return "", fmt.Errorf("invalid CustomResourceDefinition, no storage version")
+}
+
+func IsStoredVersion(crd *CustomResourceDefinition, version string) bool {
+	for _, v := range crd.Status.StoredVersions {
+		if version == v {
+			return true
+		}
+	}
+	return false
+}

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/types.go

@@ -25,6 +25,9 @@ type CustomResourceDefinitionSpec struct {
 	// Group is the group this resource belongs in
 	Group string
 	// Version is the version this resource belongs in
+	// Should be always first item in Versions field if provided.
+	// Optional, but at least one of Version or Versions must be set.
+	// Deprecated: Please use `Versions`.
 	Version string
 	// Names are the names used to describe this custom resource
 	Names CustomResourceDefinitionNames
@@ -34,6 +37,27 @@ type CustomResourceDefinitionSpec struct {
 	Validation *CustomResourceValidation
 	// Subresources describes the subresources for CustomResources
 	Subresources *CustomResourceSubresources
+	// Versions is the list of all supported versions for this resource.
+	// If Version field is provided, this field is optional.
+	// Validation: All versions must use the same validation schema for now. i.e., top
+	// level Validation field is applied to all of these versions.
+	// Order: The version name will be used to compute the order.
+	// If the version string is "kube-like", it will sort above non "kube-like" version strings, which are ordered
+	// lexicographically. "Kube-like" versions start with a "v", then are followed by a number (the major version),
+	// then optionally the string "alpha" or "beta" and another number (the minor version). These are sorted first
+	// by GA > beta > alpha, and then by comparing major version, then minor version. An example sorted list of
+	// versions: v10, v2, v1, v11beta2, v10beta3, v3beta1, v12alpha1, v11alpha2, foo1, foo10.
+	Versions []CustomResourceDefinitionVersion
+}
+
+type CustomResourceDefinitionVersion struct {
+	// Name is the version name, e.g. “v1”, “v2beta1”, etc.
+	Name string
+	// Served is a flag enabling/disabling this version from being served via REST APIs
+	Served bool
+	// Storage flags the version as storage version. There must be exactly one flagged
+	// as storage version.
+	Storage bool
 }
 
 // CustomResourceDefinitionNames indicates the names to serve this CustomResourceDefinition
@@ -115,6 +139,14 @@ type CustomResourceDefinitionStatus struct {
 	// AcceptedNames are the names that are actually being used to serve discovery
 	// They may be different than the names in spec.
 	AcceptedNames CustomResourceDefinitionNames
+
+	// StoredVersions are all versions of CustomResources that were ever persisted. Tracking these
+	// versions allows a migration path for stored versions in etcd. The field is mutable
+	// so the migration controller can first finish a migration to another version (i.e.
+	// that no old objects are left in the storage), and then remove the rest of the
+	// versions from this list.
+	// None of the versions in this list can be removed from the spec.Versions field.
+	StoredVersions []string
 }
 
 // CustomResourceCleanupFinalizer is the name of the finalizer which will delete instances of

staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1/defaults.go

@@ -31,6 +31,14 @@ func addDefaultingFuncs(scheme *runtime.Scheme) error {
 
 func SetDefaults_CustomResourceDefinition(obj *CustomResourceDefinition) {
 	SetDefaults_CustomResourceDefinitionSpec(&obj.Spec)
+	if len(obj.Status.StoredVersions) == 0 {
+		for _, v := range obj.Spec.Versions {
+			if v.Storage {
+				obj.Status.StoredVersions = append(obj.Status.StoredVersions, v.Name)
+				break
+			}
+		}
+	}
 }
 
 func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec) {
@@ -43,4 +51,16 @@ func SetDefaults_CustomResourceDefinitionSpec(obj *CustomResourceDefinitionSpec)
 	if len(obj.Names.ListKind) == 0 && len(obj.Names.Kind) > 0 {
 		obj.Names.ListKind = obj.Names.Kind + "List"
 	}
+	// If there is no list of versions, create on using deprecated Version field.
+	if len(obj.Versions) == 0 && len(obj.Version) != 0 {
+		obj.Versions = []CustomResourceDefinitionVersion{{
+			Name:    obj.Version,
+			Storage: true,
+			Served:  true,
+		}}
+	}
+	// For backward compatibility set the version field to the first item in versions list.
+	if len(obj.Version) == 0 && len(obj.Versions) != 0 {
+		obj.Version = obj.Versions[0].Name
+	}
 }