Register CSI CRDs as addon

cluster/addons/storage-crds/OWNERS

@@ -0,0 +1,6 @@
+approvers:
+- saad-ali
+- jsafrane
+- msau42
+reviewers:
+- davidz627

cluster/addons/storage-crds/README.md

@@ -0,0 +1,13 @@
+# Kubernetes CSI CRDs
+
+The Kubernetes Container Storage Interface implementation defines some API objects as CRDs that Kubernetes components
+including the Attach/Detach controller depend on.
+
+If you are using CSI, it is recommended that you enable the relevant feature gates (e.g. `CSIDriverRegistry`, `CSINodeInfo`, etc.), and ensure the CRDs in this directory are installed.
+
+These objects and their CRDs are defined in `staging/src/k8s.io/csi-api/pkg/crd/manifests`, the source of truth.
+They are copied from that CRD manifest directory to this addon directory.
+A unit test in `staging/src/k8s.io/csi-api/pkg/crd` verifies that this (and any other) copies of the manifest outside of `staging/src/k8s.io/csi-api/pkg/crd/manifests` do not drift from that source of truth.
+If you need to make changes please make changes in the `staging/src/k8s.io/csi-api/pkg/crd/manifests` directory and then update this copy.
+
+For more information, see: https://kubernetes-csi.github.io/docs/

staging/src/k8s.io/csi-api/pkg/crd/testdata/csidriver.yaml → cluster/addons/storage-crds/csidriver.yaml

@@ -1,8 +1,9 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
-  creationTimestamp: null
   name: csidrivers.csi.storage.k8s.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   group: csi.storage.k8s.io
   names:
@@ -25,9 +26,3 @@ spec:
                 information (like podName, podUID, etc.) during mount operations.
               type: string
   version: v1alpha1
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: null
-  storedVersions: null

staging/src/k8s.io/csi-api/pkg/crd/testdata/csinodeinfo.yaml → cluster/addons/storage-crds/csinodeinfo.yaml

@@ -1,8 +1,9 @@
 apiVersion: apiextensions.k8s.io/v1beta1
 kind: CustomResourceDefinition
 metadata:
-  creationTimestamp: null
   name: csinodeinfos.csi.storage.k8s.io
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
 spec:
   group: csi.storage.k8s.io
   names:
@@ -29,9 +30,3 @@ spec:
                 type: array
           type: array
   version: v1alpha1
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: null
-  storedVersions: null

cluster/gce/config-default.sh

@@ -246,10 +246,6 @@ fi
 # Optional: customize runtime config
 RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-}"
 
-if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
-  RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}"
-fi
-
 # Optional: set feature gates
 FEATURE_GATES="${KUBE_FEATURE_GATES:-ExperimentalCriticalPodAnnotation=true}"
 
@@ -260,6 +256,18 @@ if [[ ! -z "${NODE_ACCELERATORS}" ]]; then
     fi
 fi
 
+if [[ "${KUBE_FEATURE_GATES:-}" == "AllAlpha=true" ]]; then
+  RUNTIME_CONFIG="${KUBE_RUNTIME_CONFIG:-api/all=true}"
+  if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSIDriverRegistry" ]]; then
+    # If not explicitly specified, default to true.
+    FEATURE_GATES="${FEATURE_GATES},CSIDriverRegistry=true"
+  fi
+  if ! [[ "${KUBE_FEATURE_GATES:-}" =~ "CSINodeInfo" ]]; then
+    # If not explicitly specified, default to true.
+    FEATURE_GATES="${FEATURE_GATES},CSINodeInfo=true"
+  fi
+fi
+
 # Optional: Install cluster DNS.
 # Set CLUSTER_DNS_CORE_DNS to 'false' to install kube-dns instead of CoreDNS.
 CLUSTER_DNS_CORE_DNS="${CLUSTER_DNS_CORE_DNS:-true}"

cluster/gce/gci/configure-helper.sh

@@ -2567,6 +2567,9 @@ EOF
   if [[ "${ENABLE_DEFAULT_STORAGE_CLASS:-}" == "true" ]]; then
     setup-addon-manifests "addons" "storage-class/gce"
   fi
+  if [[ "${FEATURE_GATES:-}" =~ "CSIDriverRegistry=true" || "${FEATURE_GATES:-}" =~ "CSINodeInfo=true" ]]; then
+    setup-addon-manifests "addons" "storage-crds"
+  fi
   if [[ "${ENABLE_IP_MASQ_AGENT:-}" == "true" ]]; then
     setup-addon-manifests "addons" "ip-masq-agent"
   fi

staging/src/k8s.io/csi-api/pkg/crd/BUILD

@@ -6,25 +6,17 @@ go_library(
     importmap = "k8s.io/kubernetes/vendor/k8s.io/csi-api/pkg/crd",
     importpath = "k8s.io/csi-api/pkg/crd",
     visibility = ["//visibility:public"],
-    deps = [
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
-        "//staging/src/k8s.io/csi-api/pkg/apis/csi/v1alpha1:go_default_library",
-    ],
 )
 
 go_test(
     name = "go_default_test",
     srcs = ["crd_test.go"],
-    data = glob(["testdata/**"]),
-    embed = [":go_default_library"],
-    deps = [
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
-        "//staging/src/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library",
-        "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library",
-        "//vendor/github.com/ghodss/yaml:go_default_library",
+    data = [
+        "//cluster/addons:addon-srcs",
+        "//staging/src/k8s.io/csi-api/pkg/crd:manifests",
     ],
+    embed = [":go_default_library"],
+    deps = ["//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library"],
 )
 
 filegroup(
@@ -34,6 +26,14 @@ filegroup(
     visibility = ["//visibility:private"],
 )
 
+filegroup(
+    name = "csi-manifests",
+    srcs = [
+        "//staging/src/k8s.io/csi-api/pkg/crd:manifests",
+    ],
+    visibility = ["//visibility:public"],
+)
+
 filegroup(
     name = "all-srcs",
     srcs = [":package-srcs"],

staging/src/k8s.io/csi-api/pkg/crd/crd.go

@@ -14,106 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// Package crd is only for running tests to verify the manifest files
+// in this package and the addons are in sync.
+// The manifest file is currently manually generated, in the future, we
+// should invest in tooling that will automatically generate the CRD
+// manifest from the CR schema.
 package crd
-
-import (
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	csiapiv1alpha1 "k8s.io/csi-api/pkg/apis/csi/v1alpha1"
-	"reflect"
-)
-
-// NOTE: the CRD functions here and the associated unit tests are non-ideal temporary measures in
-// release 1.12 in order to aid manual CRD installation. This installation will be automated in
-// subsequent releases and as a result this package will be removed.
-
-// CSIDriverCRD returns the CustomResourceDefinition for CSIDriver object.
-func CSIDriverCRD() *apiextensionsv1beta1.CustomResourceDefinition {
-	return &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: csiapiv1alpha1.CsiDriverResourcePlural + "." + csiapiv1alpha1.GroupName,
-		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   csiapiv1alpha1.GroupName,
-			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
-			Scope:   apiextensionsv1beta1.ClusterScoped,
-			Validation: &apiextensionsv1beta1.CustomResourceValidation{
-				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
-					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-						"spec": {
-							Description: "Specification of the CSI Driver.",
-							Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-								"attachRequired": {
-									Description: "Indicates this CSI volume driver requires an attach operation," +
-										" and that Kubernetes should call attach and wait for any attach operation to" +
-										" complete before proceeding to mount.",
-									Type: "boolean",
-								},
-								"podInfoOnMountVersion": {
-									Description: "Indicates this CSI volume driver requires additional pod" +
-										" information (like podName, podUID, etc.) during mount operations.",
-									Type: "string",
-								},
-							},
-						},
-					},
-				},
-			},
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural: csiapiv1alpha1.CsiDriverResourcePlural,
-				Kind:   reflect.TypeOf(csiapiv1alpha1.CSIDriver{}).Name(),
-			},
-		},
-	}
-}
-
-// CSINodeInfoCRD returns the CustomResourceDefinition for CSINodeInfo object.
-func CSINodeInfoCRD() *apiextensionsv1beta1.CustomResourceDefinition {
-	return &apiextensionsv1beta1.CustomResourceDefinition{
-		ObjectMeta: metav1.ObjectMeta{
-			Name: csiapiv1alpha1.CsiNodeInfoResourcePlural + "." + csiapiv1alpha1.GroupName,
-		},
-		Spec: apiextensionsv1beta1.CustomResourceDefinitionSpec{
-			Group:   csiapiv1alpha1.GroupName,
-			Version: csiapiv1alpha1.SchemeGroupVersion.Version,
-			Scope:   apiextensionsv1beta1.ClusterScoped,
-			Validation: &apiextensionsv1beta1.CustomResourceValidation{
-				OpenAPIV3Schema: &apiextensionsv1beta1.JSONSchemaProps{
-					Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-						"csiDrivers": {
-							Description: "List of CSI drivers running on the node and their properties.",
-							Type:        "array",
-							Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
-								Schema: &apiextensionsv1beta1.JSONSchemaProps{
-									Properties: map[string]apiextensionsv1beta1.JSONSchemaProps{
-										"driver": {
-											Description: "The CSI driver that this object refers to.",
-											Type:        "string",
-										},
-										"nodeID": {
-											Description: "The node from the driver point of view.",
-											Type:        "string",
-										},
-										"topologyKeys": {
-											Description: "List of keys supported by the driver.",
-											Type:        "array",
-											Items: &apiextensionsv1beta1.JSONSchemaPropsOrArray{
-												Schema: &apiextensionsv1beta1.JSONSchemaProps{
-													Type: "string",
-												},
-											},
-										},
-									},
-								},
-							},
-						},
-					},
-				},
-			},
-			Names: apiextensionsv1beta1.CustomResourceDefinitionNames{
-				Plural: csiapiv1alpha1.CsiNodeInfoResourcePlural,
-				Kind:   reflect.TypeOf(csiapiv1alpha1.CSINodeInfo{}).Name(),
-			},
-		},
-	}
-}

staging/src/k8s.io/csi-api/pkg/crd/crd_test.go

@@ -14,56 +14,69 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// These tests verify the manifest files in this package and the
+// addons directory are in sync.
 package crd_test
 
 import (
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/ghodss/yaml"
-	"io/ioutil"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	apiextensionsscheme "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/scheme"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/diff"
-	"k8s.io/csi-api/pkg/crd"
-	"os"
 )
 
 func TestBootstrapCRDs(t *testing.T) {
-	testObjects(t, crd.CSIDriverCRD(), "csidriver.yaml")
-	testObjects(t, crd.CSINodeInfoCRD(), "csinodeinfo.yaml")
+	verifyCopiesAreInSync(
+		t,
+		"csidriver.yaml", /* filename */
+		"manifests",      /* sourceOfTruthDir */
+		[]string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */
+	)
+	verifyCopiesAreInSync(
+		t,
+		"csinodeinfo.yaml", /* filename */
+		"manifests",        /* sourceOfTruthDir */
+		[]string{"../../../../../../cluster/addons/storage-crds"}, /* copyDirs */
+	)
 }
 
-func testObjects(t *testing.T, crd *apiextensionsv1beta1.CustomResourceDefinition, fixtureFilename string) {
-	filename := filepath.Join("testdata", fixtureFilename)
-	expectedYAML, err := ioutil.ReadFile(filename)
-	if err != nil {
-		t.Fatal(err)
-	}
+// verifyCopiesAreInSync fails if any copies are different from source of truth.
+func verifyCopiesAreInSync(t *testing.T, filename string, sourceOfTruthDir string, copyDirs []string) {
+	sourceOfTruthFilename := filepath.Join(sourceOfTruthDir, filename)
 
-	jsonData, err := runtime.Encode(apiextensionsscheme.Codecs.LegacyCodec(apiextensionsv1beta1.SchemeGroupVersion), crd)
-	if err != nil {
-		t.Fatal(err)
+	if len(copyDirs) <= 0 {
+		t.Fatalf("copyDirs is empty. There are no copies to validate.")
 	}
-	yamlData, err := yaml.JSONToYAML(jsonData)
+
+	expectedYAML, err := ioutil.ReadFile(sourceOfTruthFilename)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if string(yamlData) != string(expectedYAML) {
-		t.Errorf("Bootstrap CRD data does not match the test fixture in %s", filename)
 
-		const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA"
-		if os.Getenv(updateEnvVar) == "true" {
-			if err := ioutil.WriteFile(filename, []byte(yamlData), os.FileMode(0755)); err == nil {
-				t.Logf("Updated data in %s", filename)
-				t.Logf("Verify the diff, commit changes, and rerun the tests")
+	for _, copyDir := range copyDirs {
+		copyFilename := filepath.Join(copyDir, filename)
+		actualYAML, err := ioutil.ReadFile(copyFilename)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if string(actualYAML) != string(expectedYAML) {
+			t.Errorf("Data in %q does not match source of truth in %q.", copyFilename, sourceOfTruthFilename)
+
+			const updateEnvVar = "UPDATE_CSI_CRD_FIXTURE_DATA"
+			if os.Getenv(updateEnvVar) == "true" {
+				if err := ioutil.WriteFile(copyFilename, []byte(expectedYAML), os.FileMode(0755)); err == nil {
+					t.Logf("Updated data in %s", copyFilename)
+					t.Logf("Verify the diff, commit changes, and rerun the tests")
+				} else {
+					t.Logf("Could not update data in %s: %v", copyFilename, err)
+				}
 			} else {
-				t.Logf("Could not update data in %s: %v", filename, err)
+				t.Logf("Diff between source of truth data and copy data in %s:\n-------------\n%s", copyFilename, diff.StringDiff(string(actualYAML), string(expectedYAML)))
+				t.Logf("If the change is expected, re-run with %s=true to update the copy data", updateEnvVar)
 			}
-		} else {
-			t.Logf("Diff between data in code and fixture data in %s:\n-------------\n%s", filename, diff.StringDiff(string(yamlData), string(expectedYAML)))
-			t.Logf("If the change is expected, re-run with %s=true to update the fixtures", updateEnvVar)
 		}
 	}
 }