Merge pull request #100 from marun/crd-cabundle-injection

Enable injection of service CA bundle for CRDs

README.md

@@ -61,8 +61,8 @@ metadata:
   uid: afee501b-b61c-11e8-833b-c85b762603b0
 ```
 
-* **apiservice cabundle injector:**
-  * Watches for apiservices annotated with 'service.beta.openshift.io/inject-cabundle=true' and updates the apiservice spec.caBundle with a base64url-encoded CA signing bundle. This is simply an apiservice variant of the above configmap injection feature.
+* **generic cabundle injector:**
+  * Watches for apiservices and crds annotated with 'service.beta.openshift.io/inject-cabundle=true' and sets the appropriate ca bundle field (apiservice -> spec.caBundle, spec.conversion.webhook.clientConfig.caBundle) with a base64url-encoded CA signing bundle. The following example is for apiservices:
 
 ```
 $ oc get apiservice/v1.build.openshift.io -o yaml

bindata/v4.0.0/controller/clusterrole.yaml

@@ -24,6 +24,15 @@ rules:
   - watch
   - update
   - patch
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
+  - update
 - apiGroups:
   - apiregistration.k8s.io
   resources:

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	go.uber.org/atomic v1.4.0 // indirect
 	k8s.io/api v0.17.0
+	k8s.io/apiextensions-apiserver v0.17.0
 	k8s.io/apimachinery v0.17.0
 	k8s.io/client-go v0.17.0
 	k8s.io/component-base v0.17.0

pkg/controller/cabundleinjector/crd.go

@@ -0,0 +1,67 @@
+package cabundleinjector
+
+import (
+	"bytes"
+
+	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	apiextclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
+	apiextinformer "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions"
+	apiextlister "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog"
+)
+
+type crdCABundleInjector struct {
+	client   apiextclientv1.CustomResourceDefinitionInterface
+	lister   apiextlister.CustomResourceDefinitionLister
+	caBundle []byte
+}
+
+func newCRDInjectorConfig(config *caBundleInjectorConfig) controllerConfig {
+	client := apiextclient.NewForConfigOrDie(config.config)
+	informers := apiextinformer.NewSharedInformerFactory(client, config.defaultResync)
+	informer := informers.Apiextensions().V1().CustomResourceDefinitions()
+	keySyncer := &crdCABundleInjector{
+		client:   client.ApiextensionsV1().CustomResourceDefinitions(),
+		lister:   informer.Lister(),
+		caBundle: config.caBundle,
+	}
+	return controllerConfig{
+		name:           "CRDCABundleInjector",
+		keySyncer:      keySyncer,
+		informerGetter: informer,
+		startInformers: func(stopChan <-chan struct{}) {
+			informers.Start(stopChan)
+		},
+	}
+}
+
+func (bi *crdCABundleInjector) Key(namespace, name string) (metav1.Object, error) {
+	return bi.lister.Get(name)
+}
+
+func (bi *crdCABundleInjector) Sync(obj metav1.Object) error {
+	crd := obj.(*apiext.CustomResourceDefinition)
+
+	if crd.Spec.Conversion == nil {
+		klog.Warningf("customresourcedefinition %s is annotated for ca bundle injection but spec.conversion is not specified", crd.Name)
+		return nil
+	}
+	if crd.Spec.Conversion.Strategy != apiext.WebhookConverter {
+		klog.Warningf("customresourcedefinition %s is annotated for ca bundle injection but does not use strategy %q", crd.Name, apiext.WebhookConverter)
+		return nil
+	}
+	if bytes.Equal(crd.Spec.Conversion.Webhook.ClientConfig.CABundle, bi.caBundle) {
+		// up-to-date
+		return nil
+	}
+
+	klog.Infof("updating customresourcedefinition %s conversion webhook config with the service signing CA bundle", crd.Name)
+
+	// make a copy to avoid mutating cache state
+	crdCopy := crd.DeepCopy()
+	crdCopy.Spec.Conversion.Webhook.ClientConfig.CABundle = bi.caBundle
+	_, err := bi.client.Update(crdCopy)
+	return err
+}

pkg/controller/cabundleinjector/starter.go

@@ -60,6 +60,7 @@ func StartCABundleInjector(ctx context.Context, controllerContext *controllercmd
 	configConstructors := []configBuilderFunc{
 		newAPIServiceInjectorConfig,
 		newConfigMapInjectorConfig,
+		newCRDInjectorConfig,
 	}
 	controllerRunners := []controller.Runner{}
 	for _, configConstructor := range configConstructors {

test/e2e/e2e_test.go

@@ -16,6 +16,8 @@ import (
 	"github.com/prometheus/common/model"
 
 	v1 "k8s.io/api/core/v1"
+	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kruntime "k8s.io/apimachinery/pkg/runtime"
@@ -612,6 +614,34 @@ func pollForAPIService(t *testing.T, client apiserviceclientv1.APIServiceInterfa
 	return obj.(*apiregv1.APIService), nil
 }
 
+// pollForCRD returns the specified CustomResourceDefinition if the ca
+// bundle for its conversion webhook config matches the provided value
+// before the polling timeout.
+func pollForCRD(t *testing.T, client apiextclient.CustomResourceDefinitionInterface, name string, expectedCABundle []byte) (*apiext.CustomResourceDefinition, error) {
+	resourceID := fmt.Sprintf("CustomResourceDefinition %q", name)
+	obj, err := pollForResource(t, resourceID, pollTimeout, func() (kruntime.Object, error) {
+		crd, err := client.Get(name, metav1.GetOptions{})
+		if err != nil {
+			return nil, err
+		}
+		if crd.Spec.Conversion == nil || crd.Spec.Conversion.Webhook == nil || crd.Spec.Conversion.Webhook.ClientConfig == nil {
+			return nil, fmt.Errorf("spec.conversion.webhook.webhook.clientConfig not set")
+		}
+		actualCABundle := crd.Spec.Conversion.Webhook.ClientConfig.CABundle
+		if len(actualCABundle) == 0 {
+			return nil, fmt.Errorf("ca bundle not injected")
+		}
+		if !bytes.Equal(actualCABundle, expectedCABundle) {
+			return nil, fmt.Errorf("ca bundle does match the expected value")
+		}
+		return crd, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return obj.(*apiext.CustomResourceDefinition), nil
+}
+
 // setInjectionAnnotation sets the annotation that will trigger the
 // injection of a ca bundle.
 func setInjectionAnnotation(objMeta *metav1.ObjectMeta) {
@@ -1026,6 +1056,93 @@ func TestE2E(t *testing.T) {
 			t.Fatalf("error waiting for ca bundle to be re-injected: %v", err)
 		}
 	})
+
+	t.Run("crd-ca-bundle-injection", func(t *testing.T) {
+		client := apiextclient.NewForConfigOrDie(adminConfig).CustomResourceDefinitions()
+
+		// Create a crd with the injection annotation
+		randomGroup := fmt.Sprintf("e2e-%s.example.com", randSeq(10))
+		pluralName := "cabundleinjectiontargets"
+		version := "v1beta1"
+		obj := &apiext.CustomResourceDefinition{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: fmt.Sprintf("%s.%s", pluralName, randomGroup),
+			},
+			Spec: apiext.CustomResourceDefinitionSpec{
+				Group: randomGroup,
+				Scope: apiext.ClusterScoped,
+				Names: apiext.CustomResourceDefinitionNames{
+					Plural: pluralName,
+					Kind:   "CABundleInjectionTarget",
+				},
+				Conversion: &apiext.CustomResourceConversion{
+					// CA bundle will only be injected for a webhook converter
+					Strategy: apiext.WebhookConverter,
+					Webhook: &apiext.WebhookConversion{
+						// CA bundle will be set on the following struct
+						ClientConfig: &apiext.WebhookClientConfig{
+							Service: &apiext.ServiceReference{
+								Namespace: "foo",
+								Name:      "foo",
+							},
+						},
+						ConversionReviewVersions: []string{
+							version,
+						},
+					},
+				},
+				// At least one version must be defined for a v1 crd to be valid
+				Versions: []apiext.CustomResourceDefinitionVersion{
+					{
+						Name:    version,
+						Storage: true,
+						Schema: &apiext.CustomResourceValidation{
+							OpenAPIV3Schema: &apiext.JSONSchemaProps{
+								Type: "object",
+							},
+						},
+					},
+				},
+			},
+		}
+		setInjectionAnnotation(&obj.ObjectMeta)
+		createdObj, err := client.Create(obj)
+		if err != nil {
+			t.Fatalf("error creating crd: %v", err)
+		}
+		defer func() {
+			err := client.Delete(obj.Name, &metav1.DeleteOptions{})
+			if err != nil {
+				t.Errorf("Failed to cleanup crd: %v", err)
+			}
+		}()
+
+		// Retrieve the expected CA bundle
+		expectedCABundle, err := pollForSigningCABundle(t, adminClient)
+		if err != nil {
+			t.Fatalf("error retrieving the signing ca bundle: %v", err)
+		}
+
+		// Wait for the expected bundle to be injected
+		injectedObj, err := pollForCRD(t, client, createdObj.Name, expectedCABundle)
+		if err != nil {
+			t.Fatalf("error waiting for ca bundle to be injected: %v", err)
+		}
+
+		// Set an invalid ca bundle
+		whClientConfig := injectedObj.Spec.Conversion.Webhook.ClientConfig
+		whClientConfig.CABundle = append(whClientConfig.CABundle, []byte("garbage")...)
+		_, err = client.Update(injectedObj)
+		if err != nil {
+			t.Fatalf("error updated crd: %v", err)
+		}
+
+		// Check that the expected ca bundle is restored
+		_, err = pollForCRD(t, client, createdObj.Name, expectedCABundle)
+		if err != nil {
+			t.Fatalf("error waiting for ca bundle to be re-injected: %v", err)
+		}
+	})
 }
 
 func init() {