diff --git a/pkg/envtest/crd.go b/pkg/envtest/crd.go index 04dc5fc2d5..1a49012852 100644 --- a/pkg/envtest/crd.go +++ b/pkg/envtest/crd.go @@ -25,6 +25,7 @@ import ( "os" "path/filepath" "time" + "fmt" "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -60,27 +61,71 @@ type CRDInstallOptions struct { // uninstalled when terminating the test environment. // Defaults to false. CleanUpAfterUse bool + + // Used for CRD Conversoin Webhooks + + *LocalWebhookOptions } const defaultPollInterval = 100 * time.Millisecond const defaultMaxWait = 10 * time.Second +func ConfigureCRDWebhooks(opts *LocalWebhookOptions, crds []runtime.Object) error { + caData, err := opts.setupCAIfNecessary() + if err != nil { + return fmt.Errorf("unable to initialize CA for CRD conversion webhook serving certs: %w", err) + } + hostPort, err := opts.HostPort() + if err != nil { + return fmt.Errorf("unable to grab host-port for CRD conversion webhooks: %w", err) + } + + for i, crd := range runtimeCRDListToUnstructured(crds) { + log := log.WithValues("crd", crd.GetName()) + conv, found, err := unstructured.NestedMap(crd.Object, "spec", "conversion") + if !found || err != nil { + log.V(1).Info("not configuring conversion webhooks on CRD, no conversion") + continue + } + + if strat, found := conv["strategy"].(string); !found || strat != "Webhook" { + log.V(1).Info("not configuring conversion webhooks on CRD, strategy not already webhook", "strategy", strat) + continue + } + + if _, found := conv["webhookClientConfig"].(map[string]interface{}); !found { + conv["webhookClientConfig"] = make(map[string]interface{}) + } + modifyClientConfig(conv["webhookClientConfig"].(map[string]interface{}), caData, hostPort) + unstructured.SetNestedMap(crd.Object, conv, "spec", "conversion") + log.V(1).Info("configured crd webhook", "conv", conv) + + crds[i] = crd + } + + return nil +} + // InstallCRDs installs a collection of CRDs into a cluster by reading the crd yaml files from a directory -func InstallCRDs(config *rest.Config, options CRDInstallOptions) ([]runtime.Object, error) { - defaultCRDOptions(&options) +func InstallCRDs(config *rest.Config, options *CRDInstallOptions) ([]runtime.Object, error) { + defaultCRDOptions(options) // Read the CRD yamls into options.CRDs - if err := readCRDFiles(&options); err != nil { + if err := readCRDFiles(options); err != nil { return nil, err } + if err := ConfigureCRDWebhooks(options.LocalWebhookOptions, options.CRDs); err != nil { + return options.CRDs, err + } + // Create the CRDs in the apiserver if err := CreateCRDs(config, options.CRDs); err != nil { return options.CRDs, err } // Wait for the CRDs to appear as Resources in the apiserver - if err := WaitForCRDs(config, options.CRDs, options); err != nil { + if err := WaitForCRDs(config, options.CRDs, *options); err != nil { return options.CRDs, err } @@ -108,6 +153,9 @@ func defaultCRDOptions(o *CRDInstallOptions) { if o.PollInterval == 0 { o.PollInterval = defaultPollInterval } + if o.LocalWebhookOptions == nil { + o.LocalWebhookOptions = &LocalWebhookOptions{} + } } // WaitForCRDs waits for the CRDs to appear in discovery diff --git a/pkg/envtest/server.go b/pkg/envtest/server.go index 1ca2d8eadc..969a622567 100644 --- a/pkg/envtest/server.go +++ b/pkg/envtest/server.go @@ -255,13 +255,14 @@ func (te *Environment) Start() (*rest.Config, error) { te.CRDInstallOptions.CRDs = mergeCRDs(te.CRDInstallOptions.CRDs, te.CRDs) te.CRDInstallOptions.Paths = mergePaths(te.CRDInstallOptions.Paths, te.CRDDirectoryPaths) te.CRDInstallOptions.ErrorIfPathMissing = te.ErrorIfCRDPathMissing - crds, err := InstallCRDs(te.Config, te.CRDInstallOptions) + crds, err := InstallCRDs(te.Config, &te.CRDInstallOptions) if err != nil { return te.Config, err } te.CRDs = crds log.V(1).Info("installing webhooks") + te.WebhookInstallOptions.LocalWebhookOptions = te.CRDInstallOptions.LocalWebhookOptions err = te.WebhookInstallOptions.Install(te.Config) return te.Config, err diff --git a/pkg/envtest/webhook.go b/pkg/envtest/webhook.go index dd6774d0cf..f03aab3612 100644 --- a/pkg/envtest/webhook.go +++ b/pkg/envtest/webhook.go @@ -38,6 +38,24 @@ import ( "sigs.k8s.io/yaml" ) +type LocalWebhookOptions struct { + // LocalServingHost is the host for serving webhooks on. + // it will be automatically populated + LocalServingHost string + + // LocalServingPort is the allocated port for serving webhooks on. + // it will be automatically populated by a random available local port + LocalServingPort int + + // LocalServingCertDir is the allocated directory for serving certificates. + // it will be automatically populated by the local temp dir + LocalServingCertDir string + + // The CA data corresponding to the serving certs generated for the webhooks. + // It will be set automatically. + CAData []byte +} + // WebhookInstallOptions are the options for installing mutating or validating webhooks type WebhookInstallOptions struct { // Paths is a list of paths to the directories containing the mutating or validating webhooks yaml or json configs. @@ -52,30 +70,30 @@ type WebhookInstallOptions struct { // IgnoreErrorIfPathMissing will ignore an error if a DirectoryPath does not exist when set to true IgnoreErrorIfPathMissing bool - // LocalServingHost is the host for serving webhooks on. - // it will be automatically populated - LocalServingHost string - - // LocalServingPort is the allocated port for serving webhooks on. - // it will be automatically populated by a random available local port - LocalServingPort int - - // LocalServingCertDir is the allocated directory for serving certificates. - // it will be automatically populated by the local temp dir - LocalServingCertDir string // MaxTime is the max time to wait MaxTime time.Duration // PollInterval is the interval to check PollInterval time.Duration + + *LocalWebhookOptions } // ModifyWebhookDefinitions modifies webhook definitions by: // - applying CABundle based on the provided tinyca // - if webhook client config uses service spec, it's removed and replaced with direct url -func (o *WebhookInstallOptions) ModifyWebhookDefinitions(caData []byte) error { - hostPort, err := o.generateHostPort() +func (o *WebhookInstallOptions) ModifyWebhookDefinitions() error { + if o.LocalWebhookOptions == nil { + o.LocalWebhookOptions = &LocalWebhookOptions{} + } + + caData, err := o.setupCAIfNecessary() + if err != nil { + return err + } + + hostPort, err := o.HostPort() if err != nil { return err } @@ -111,14 +129,11 @@ func (o *WebhookInstallOptions) ModifyWebhookDefinitions(caData []byte) error { o.ValidatingWebhooks[i] = unstructuredHook } } + return nil } -func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort string) (map[string]interface{}, error) { - clientConfig, found, err := unstructured.NestedMap(webhook, "clientConfig") - if !found || err != nil { - return nil, fmt.Errorf("cannot find clientconfig: %v", err) - } +func modifyClientConfig(clientConfig map[string]interface{}, caData []byte, hostPort string) { clientConfig["caBundle"] = base64.StdEncoding.EncodeToString(caData) servicePath, found, err := unstructured.NestedString(clientConfig, "service", "path") if found && err == nil { @@ -129,32 +144,44 @@ func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort strin clientConfig["url"] = url clientConfig["service"] = nil } +} + +func modifyWebhook(webhook map[string]interface{}, caData []byte, hostPort string) (map[string]interface{}, error) { + clientConfig, found, err := unstructured.NestedMap(webhook, "clientConfig") + if !found || err != nil { + return nil, fmt.Errorf("cannot find clientconfig: %v", err) + } + modifyClientConfig(clientConfig, caData, hostPort) webhook["clientConfig"] = clientConfig return webhook, nil } -func (o *WebhookInstallOptions) generateHostPort() (string, error) { +func (o *LocalWebhookOptions) generateHostPort() error { port, host, err := addr.Suggest() if err != nil { - return "", fmt.Errorf("unable to grab random port for serving webhooks on: %v", err) + return fmt.Errorf("unable to grab random port for serving webhooks on: %v", err) } o.LocalServingPort = port o.LocalServingHost = host - return net.JoinHostPort(host, fmt.Sprintf("%d", port)), nil + return nil +} + +func (o *LocalWebhookOptions) HostPort() (string, error) { + if o.LocalServingPort == 0 { + if err := o.generateHostPort(); err != nil { + return "", err + } + } + return net.JoinHostPort(o.LocalServingHost, fmt.Sprintf("%d", o.LocalServingPort)), nil } // Install installs specified webhooks to the API server func (o *WebhookInstallOptions) Install(config *rest.Config) error { - hookCA, err := o.setupCA() - if err != nil { - return err - } if err := parseWebhookDirs(o); err != nil { return err } - err = o.ModifyWebhookDefinitions(hookCA) - if err != nil { + if err := o.ModifyWebhookDefinitions(); err != nil { return err } @@ -169,14 +196,19 @@ func (o *WebhookInstallOptions) Install(config *rest.Config) error { return nil } -// Cleanup cleans up cert directories -func (o *WebhookInstallOptions) Cleanup() error { +func (o *LocalWebhookOptions) cleanupCertsIfNecessary() error { if o.LocalServingCertDir != "" { return os.RemoveAll(o.LocalServingCertDir) } + o.LocalServingCertDir = "" return nil } +// Cleanup cleans up cert directories +func (o *WebhookInstallOptions) Cleanup() error { + return o.cleanupCertsIfNecessary() +} + // WaitForWebhooks waits for the Webhooks to be available through API server func WaitForWebhooks(config *rest.Config, mutatingWebhooks []runtime.Object, @@ -243,8 +275,15 @@ func (p *webhookPoller) poll() (done bool, err error) { return allFound, nil } +func (o *LocalWebhookOptions) setupCAIfNecessary() ([]byte, error) { + if o.CAData != nil { + return o.CAData, nil + } + return o.setupCA() +} + // setupCA creates CA for testing and writes them to disk -func (o *WebhookInstallOptions) setupCA() ([]byte, error) { +func (o *LocalWebhookOptions) setupCA() ([]byte, error) { hookCA, err := integration.NewTinyCA() if err != nil { return nil, fmt.Errorf("unable to set up webhook CA: %v", err) @@ -273,6 +312,8 @@ func (o *WebhookInstallOptions) setupCA() ([]byte, error) { return nil, fmt.Errorf("unable to write webhook serving key to disk: %v", err) } + o.CAData = certData + return certData, nil }