main.go

/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"github.com/openshift/hypershift/support/globalconfig"

	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"

	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/go-logr/logr"
	configapi "github.com/openshift/api/config/v1"
	operatorv1 "github.com/openshift/api/operator/v1"
	hyperapi "github.com/openshift/hypershift/api"
	hyperv1 "github.com/openshift/hypershift/api/v1beta1"
	awsutil "github.com/openshift/hypershift/cmd/infra/aws/util"
	"github.com/openshift/hypershift/cmd/util"
	"github.com/openshift/hypershift/hypershift-operator/controllers/hostedcluster"
	"github.com/openshift/hypershift/hypershift-operator/controllers/nodepool"
	"github.com/openshift/hypershift/hypershift-operator/controllers/platform/aws"
	"github.com/openshift/hypershift/hypershift-operator/controllers/proxy"
	"github.com/openshift/hypershift/hypershift-operator/controllers/scheduler"
	"github.com/openshift/hypershift/hypershift-operator/controllers/supportedversion"
	"github.com/openshift/hypershift/hypershift-operator/controllers/uwmtelemetry"
	kvinfra "github.com/openshift/hypershift/kubevirtexternalinfra"
	"github.com/openshift/hypershift/pkg/version"
	"github.com/openshift/hypershift/support/capabilities"
	"github.com/openshift/hypershift/support/metrics"
	"github.com/openshift/hypershift/support/releaseinfo"
	"github.com/openshift/hypershift/support/upsert"
	hyperutil "github.com/openshift/hypershift/support/util"
	"github.com/spf13/cobra"
	"go.uber.org/zap/zapcore"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/discovery"
	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
	ctrl "sigs.k8s.io/controller-runtime"
	crclient "sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
	"sigs.k8s.io/controller-runtime/pkg/log/zap"
)

func main() {
	ctrl.SetLogger(zap.New(zap.UseDevMode(true), zap.JSONEncoder(func(o *zapcore.EncoderConfig) {
		o.EncodeTime = zapcore.RFC3339TimeEncoder
	})))

	cmd := &cobra.Command{
		Use: "hypershift-operator",
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Help()
			os.Exit(1)
		},
	}

	cmd.Version = version.String()

	cmd.AddCommand(NewStartCommand())
	cmd.AddCommand(NewInitCommand())

	if err := cmd.Execute(); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}

type StartOptions struct {
	Namespace                        string
	DeploymentName                   string
	PodName                          string
	MetricsAddr                      string
	CertDir                          string
	EnableOCPClusterMonitoring       bool
	EnableCIDebugOutput              bool
	ControlPlaneOperatorImage        string
	RegistryOverrides                map[string]string
	PrivatePlatform                  string
	OIDCStorageProviderS3BucketName  string
	OIDCStorageProviderS3Region      string
	OIDCStorageProviderS3Credentials string
	EnableUWMTelemetryRemoteWrite    bool
}

func NewStartCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "run",
		Short: "Runs the Hypershift operator",
	}

	opts := StartOptions{
		Namespace:                        "hypershift",
		DeploymentName:                   "operator",
		MetricsAddr:                      "0",
		CertDir:                          "",
		ControlPlaneOperatorImage:        "",
		RegistryOverrides:                map[string]string{},
		PrivatePlatform:                  string(hyperv1.NonePlatform),
		OIDCStorageProviderS3Region:      "",
		OIDCStorageProviderS3Credentials: "",
	}

	cmd.Flags().StringVar(&opts.Namespace, "namespace", opts.Namespace, "The namespace this operator lives in")
	cmd.Flags().StringVar(&opts.DeploymentName, "deployment-name", opts.DeploymentName, "Legacy flag, does nothing. Use --pod-name instead.")
	cmd.Flags().StringVar(&opts.PodName, "pod-name", opts.PodName, "The name of the pod the operator runs in")
	cmd.Flags().StringVar(&opts.MetricsAddr, "metrics-addr", opts.MetricsAddr, "The address the metric endpoint binds to.")
	cmd.Flags().StringVar(&opts.CertDir, "cert-dir", opts.CertDir, "Path to the serving key and cert for manager")
	cmd.Flags().StringVar(&opts.ControlPlaneOperatorImage, "control-plane-operator-image", opts.ControlPlaneOperatorImage, "A control plane operator image to use (defaults to match this operator if running in a deployment)")
	cmd.Flags().BoolVar(&opts.EnableOCPClusterMonitoring, "enable-ocp-cluster-monitoring", opts.EnableOCPClusterMonitoring, "Development-only option that will make your OCP cluster unsupported: If the cluster Prometheus should be configured to scrape metrics")
	cmd.Flags().BoolVar(&opts.EnableCIDebugOutput, "enable-ci-debug-output", false, "If extra CI debug output should be enabled")
	cmd.Flags().StringToStringVar(&opts.RegistryOverrides, "registry-overrides", map[string]string{}, "registry-overrides contains the source registry string as a key and the destination registry string as value. Images before being applied are scanned for the source registry string and if found the string is replaced with the destination registry string. Format is: sr1=dr1,sr2=dr2")
	cmd.Flags().StringVar(&opts.PrivatePlatform, "private-platform", opts.PrivatePlatform, "Platform on which private clusters are supported by this operator (supports \"AWS\" or \"None\")")
	cmd.Flags().StringVar(&opts.OIDCStorageProviderS3BucketName, "oidc-storage-provider-s3-bucket-name", "", "Name of the bucket in which to store the clusters OIDC discovery information. Required for AWS guest clusters")
	cmd.Flags().StringVar(&opts.OIDCStorageProviderS3Region, "oidc-storage-provider-s3-region", opts.OIDCStorageProviderS3Region, "Region in which the OIDC bucket is located. Required for AWS guest clusters")
	cmd.Flags().StringVar(&opts.OIDCStorageProviderS3Credentials, "oidc-storage-provider-s3-credentials", opts.OIDCStorageProviderS3Credentials, "Location of the credentials file for the OIDC bucket. Required for AWS guest clusters.")
	cmd.Flags().BoolVar(&opts.EnableUWMTelemetryRemoteWrite, "enable-uwm-telemetry-remote-write", opts.EnableUWMTelemetryRemoteWrite, "If true, enables a controller that ensures user workload monitoring is enabled and that it is configured to remote write telemetry metrics from control planes")

	cmd.Run = func(cmd *cobra.Command, args []string) {
		ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
		defer cancel()

		switch hyperv1.PlatformType(opts.PrivatePlatform) {
		case hyperv1.AWSPlatform, hyperv1.NonePlatform:
		default:
			fmt.Println(fmt.Sprintf("Unsupported private platform: %q", opts.PrivatePlatform))
			os.Exit(1)
		}

		if err := run(ctx, &opts, ctrl.Log.WithName("setup")); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	return cmd
}

func run(ctx context.Context, opts *StartOptions, log logr.Logger) error {

	log.Info("Starting hypershift-operator-manager", "version", version.String())

	restConfig := ctrl.GetConfigOrDie()
	restConfig.UserAgent = "hypershift-operator-manager"
	leaseDuration := time.Second * 60
	renewDeadline := time.Second * 40
	retryPeriod := time.Second * 15
	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
		Scheme:                        hyperapi.Scheme,
		MetricsBindAddress:            opts.MetricsAddr,
		Port:                          9443,
		CertDir:                       opts.CertDir,
		LeaderElection:                true,
		LeaderElectionID:              "hypershift-operator-leader-elect",
		LeaderElectionResourceLock:    "leases",
		LeaderElectionReleaseOnCancel: true,
		LeaderElectionNamespace:       opts.Namespace,
		LeaseDuration:                 &leaseDuration,
		RenewDeadline:                 &renewDeadline,
		RetryPeriod:                   &retryPeriod,
	})
	if err != nil {
		return fmt.Errorf("unable to start manager: %w", err)
	}

	kubeDiscoveryClient, err := discovery.NewDiscoveryClientForConfig(mgr.GetConfig())
	if err != nil {
		return fmt.Errorf("unable to create discovery client: %w", err)
	}

	mgmtClusterCaps, err := capabilities.DetectManagementClusterCapabilities(kubeDiscoveryClient)
	if err != nil {
		return fmt.Errorf("unable to detect cluster capabilities: %w", err)
	}

	lookupOperatorImage := func(userSpecifiedImage string) (string, error) {
		if len(userSpecifiedImage) > 0 {
			log.Info("using image from arguments", "image", userSpecifiedImage)
			return userSpecifiedImage, nil
		}
		me := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: opts.Namespace, Name: opts.PodName}}
		if err := mgr.GetAPIReader().Get(ctx, crclient.ObjectKeyFromObject(me), me); err != nil {
			return "", fmt.Errorf("failed to get operator pod %s: %w", crclient.ObjectKeyFromObject(me), err)
		}
		// Use the container status to make sure we get the sha256 reference rather than a potentially
		// floating tag.
		for _, container := range me.Status.ContainerStatuses {
			// TODO: could use downward API for this too, overkill?
			if container.Name == "operator" {
				return strings.TrimPrefix(container.ImageID, "docker-pullable://"), nil
			}
		}
		return "", fmt.Errorf("couldn't locate operator container on deployment")
	}
	var operatorImage string
	if err := wait.PollImmediate(5*time.Second, 30*time.Second, func() (bool, error) {
		operatorImage, err = lookupOperatorImage(opts.ControlPlaneOperatorImage)
		if err != nil {
			return false, err
		}
		// Apparently this is occasionally set to an empty string
		if operatorImage == "" {
			log.Info("operator image is empty, retrying")
			return false, nil
		}
		return true, nil
	}); err != nil {
		if err != nil {
			return fmt.Errorf("failed to find operator image: %w", err)
		}
	}

	log.Info("using hosted control plane operator image", "operator-image", operatorImage)

	createOrUpdate := upsert.New(opts.EnableCIDebugOutput)

	metricsSet, err := metrics.MetricsSetFromEnv()
	if err != nil {
		return err
	}
	log.Info("Using metrics set", "set", metricsSet.String())

	sreConfigHash := ""
	if metricsSet == metrics.MetricsSetSRE {
		readerClient := mgr.GetAPIReader()
		cm := metrics.SREMetricsSetConfigurationConfigMap(opts.Namespace)
		err := readerClient.Get(context.Background(), crclient.ObjectKeyFromObject(cm), cm)
		if err != nil {
			if apierrors.IsNotFound(err) {
				log.Info("WARNING: no configuration found for the SRE metrics set")
			} else {
				return fmt.Errorf("unable to read SRE metrics set configmap: %w", err)
			}
		} else {
			if err := metrics.LoadSREMetricsSetConfigurationFromConfigMap(cm); err != nil {
				return fmt.Errorf("unable to load SRE metrics configuration: %w", err)
			}
			sreConfigHash = metrics.SREMetricsSetConfigHash(cm)
		}
	}

	// The mgr and therefore the cache is not started yet, thus we have to construct a client that
	// directly reads from the api.
	apiReadingClient, err := crclient.NewDelegatingClient(crclient.NewDelegatingClientInput{
		CacheReader: mgr.GetAPIReader(),
		Client:      mgr.GetClient(),
	})
	if err != nil {
		return fmt.Errorf("failed to construct api reading client: %w", err)
	}

	// Populate registry overrides with any ICSP and IDMS from a OpenShift management cluster
	var imageRegistryOverrides map[string][]string
	if mgmtClusterCaps.Has(capabilities.CapabilityICSP) || mgmtClusterCaps.Has(capabilities.CapabilityIDMS) {
		imageRegistryOverrides, err = globalconfig.GetAllImageRegistryMirrors(ctx, apiReadingClient, mgmtClusterCaps.Has(capabilities.CapabilityIDMS))
		if err != nil {
			return fmt.Errorf("failed to populate image registry overrides: %w", err)
		}
	}

	releaseProviderWithOpenShiftImageRegistryOverrides := &releaseinfo.ProviderWithOpenShiftImageRegistryOverridesDecorator{
		Delegate: &releaseinfo.RegistryMirrorProviderDecorator{
			Delegate: &releaseinfo.CachedProvider{
				Inner: &releaseinfo.RegistryClientProvider{},
				Cache: map[string]*releaseinfo.ReleaseImage{},
			},
			RegistryOverrides: opts.RegistryOverrides,
		},
		OpenShiftImageRegistryOverrides: imageRegistryOverrides,
	}

	hostedClusterReconciler := &hostedcluster.HostedClusterReconciler{
		Client:                        mgr.GetClient(),
		ManagementClusterCapabilities: mgmtClusterCaps,
		HypershiftOperatorImage:       operatorImage,
		ReleaseProvider:               releaseProviderWithOpenShiftImageRegistryOverrides,
		EnableOCPClusterMonitoring:    opts.EnableOCPClusterMonitoring,
		EnableCIDebugOutput:           opts.EnableCIDebugOutput,
		ImageMetadataProvider:         &hyperutil.RegistryClientImageMetadataProvider{},
		MetricsSet:                    metricsSet,
		OperatorNamespace:             opts.Namespace,
		SREConfigHash:                 sreConfigHash,
		KubevirtInfraClients:          kvinfra.NewKubevirtInfraClientMap(),
	}
	if opts.OIDCStorageProviderS3BucketName != "" {
		awsSession := awsutil.NewSession("hypershift-operator-oidc-bucket", opts.OIDCStorageProviderS3Credentials, "", "", opts.OIDCStorageProviderS3Region)
		awsConfig := awsutil.NewConfig()
		s3Client := s3.New(awsSession, awsConfig)
		hostedClusterReconciler.S3Client = s3Client
		hostedClusterReconciler.OIDCStorageProviderS3BucketName = opts.OIDCStorageProviderS3BucketName
	}
	if err := hostedClusterReconciler.SetupWithManager(mgr, createOrUpdate, metricsSet, opts.Namespace); err != nil {
		return fmt.Errorf("unable to create controller: %w", err)
	}
	if opts.CertDir != "" {
		if err := hostedcluster.SetupWebhookWithManager(mgr); err != nil {
			return fmt.Errorf("unable to create webhook: %w", err)
		}
	}

	// Since we dropped the validation webhook server we need to ensure this resource doesn't exist
	// otherwise it will intercept kas requests and fail.
	// TODO (alberto): dropped in 4.14.
	validatingWebhookConfiguration := &admissionregistrationv1.ValidatingWebhookConfiguration{
		TypeMeta: metav1.TypeMeta{
			Kind:       "ValidatingWebhookConfiguration",
			APIVersion: admissionregistrationv1.SchemeGroupVersion.String(),
		},
		ObjectMeta: metav1.ObjectMeta{
			Namespace: opts.Namespace,
			Name:      hyperv1.GroupVersion.Group,
		},
	}
	if err := mgr.GetClient().Delete(ctx, validatingWebhookConfiguration); err != nil {
		if !apierrors.IsNotFound(err) {
			return err
		}
	}

	if err := (&nodepool.NodePoolReconciler{
		Client:                  mgr.GetClient(),
		ReleaseProvider:         releaseProviderWithOpenShiftImageRegistryOverrides,
		CreateOrUpdateProvider:  createOrUpdate,
		HypershiftOperatorImage: operatorImage,
		ImageMetadataProvider:   &hyperutil.RegistryClientImageMetadataProvider{},
		KubevirtInfraClients:    kvinfra.NewKubevirtInfraClientMap(),
	}).SetupWithManager(mgr); err != nil {
		return fmt.Errorf("unable to create controller: %w", err)
	}

	if mgmtClusterCaps.Has(capabilities.CapabilityProxy) {
		if err := proxy.Setup(mgr, opts.Namespace, opts.DeploymentName); err != nil {
			return fmt.Errorf("failed to set up the proxy controller: %w", err)
		}
	}

	// Start platform-specific controllers
	switch hyperv1.PlatformType(opts.PrivatePlatform) {
	case hyperv1.AWSPlatform:
		if err := (&aws.AWSEndpointServiceReconciler{
			Client:                 mgr.GetClient(),
			CreateOrUpdateProvider: createOrUpdate,
		}).SetupWithManager(mgr); err != nil {
			return fmt.Errorf("unable to create controller: %w", err)
		}
	}

	// Start controller to manage supported versions configmap
	if err := supportedversion.New(mgr.GetClient(), createOrUpdate, opts.Namespace).
		SetupWithManager(mgr); err != nil {
		return fmt.Errorf("unable to create supported version controller: %w", err)
	}

	// If enabled, start controller to ensure UWM stack is enabled and configured
	// to remotely write telemetry metrics
	if opts.EnableUWMTelemetryRemoteWrite {
		if err := (&uwmtelemetry.Reconciler{
			Namespace:              opts.Namespace,
			Client:                 mgr.GetClient(),
			CreateOrUpdateProvider: createOrUpdate,
		}).SetupWithManager(mgr); err != nil {
			return fmt.Errorf("unable to create uwm telemetry controller: %w", err)
		}
	}

	// Start controllers to manage dedicated request serving isolation
	nodeReaper := scheduler.DedicatedServingComponentNodeReaper{
		Client: mgr.GetClient(),
	}
	if err := nodeReaper.SetupWithManager(mgr); err != nil {
		return fmt.Errorf("unable to create dedicated serving component node reaper controller: %w", err)
	}
	hcScheduler := scheduler.DedicatedServingComponentScheduler{
		Client: mgr.GetClient(),
	}
	if err := hcScheduler.SetupWithManager(mgr, createOrUpdate); err != nil {
		return fmt.Errorf("unable to create dedicated serving component scheduler controller: %w", err)
	}

	// If it exists, block default ingress controller from admitting HCP private routes
	ic := &operatorv1.IngressController{
		ObjectMeta: metav1.ObjectMeta{
			Name:      "default",
			Namespace: "openshift-ingress-operator",
		},
	}
	if err := apiReadingClient.Get(ctx, types.NamespacedName{Namespace: ic.Namespace, Name: ic.Name}, ic); err == nil {
		if _, err := controllerutil.CreateOrUpdate(ctx, apiReadingClient, ic, func() error {
			if ic.Spec.RouteSelector == nil {
				ic.Spec.RouteSelector = &metav1.LabelSelector{}
			}
			if ic.Spec.RouteSelector.MatchExpressions == nil {
				ic.Spec.RouteSelector.MatchExpressions = []metav1.LabelSelectorRequirement{}
			}
			for _, requirement := range ic.Spec.RouteSelector.MatchExpressions {
				if requirement.Key != hyperutil.HCPRouteLabel {
					continue
				}
				requirement.Operator = metav1.LabelSelectorOpDoesNotExist
				return nil
			}
			ic.Spec.RouteSelector.MatchExpressions = append(ic.Spec.RouteSelector.MatchExpressions, metav1.LabelSelectorRequirement{
				Key:      hyperutil.HCPRouteLabel,
				Operator: metav1.LabelSelectorOpDoesNotExist,
			})
			return nil
		}); err != nil {
			return fmt.Errorf("failed to reconcile default ingress controller: %w", err)
		}
		log.Info("reconciled default ingress controller")
	}
	if err != nil && apierrors.IsNotFound(err) {
		return fmt.Errorf("failed to get ingress controller: %w", err)
	}

	if err := setupMetrics(mgr); err != nil {
		return fmt.Errorf("failed to setup metrics: %w", err)
	}

	// Start the controllers
	log.Info("starting manager")
	return mgr.Start(ctx)
}

func NewInitCommand() *cobra.Command {
	ctrl.SetLogger(zap.New(zap.UseDevMode(true), zap.JSONEncoder(func(o *zapcore.EncoderConfig) {
		o.EncodeTime = zapcore.RFC3339TimeEncoder
	})))

	cmd := &cobra.Command{
		Use:   "init",
		Short: "Initializes the environment for the Hypershift Operator",
	}

	cmd.Run = func(cmd *cobra.Command, args []string) {
		ctx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
		defer cancel()

		if err := runInit(ctx, ctrl.Log.WithName("init")); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
	}

	return cmd
}

func runInit(ctx context.Context, log logr.Logger) error {
	log.Info("Initializing environment for Hypershift Operator")
	client, err := util.GetClient()
	if err != nil {
		return err
	}
	config, err := ctrl.GetConfig()
	if err != nil {
		return err
	}
	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return fmt.Errorf("unable to get discovery client: %w", err)
	}
	mgmtClusterCaps, err := capabilities.DetectManagementClusterCapabilities(discoveryClient)
	if err != nil {
		return fmt.Errorf("unable to detect management cluster capabilities: %w", err)
	}

	trustedCABundle := new(bytes.Buffer)
	var ocmTrustedCABundle []byte
	if _, err := os.Stat("/var/run/ca-trust/tls-ca-bundle.pem"); err == nil {
		ocmTrustedCABundle, err = ioutil.ReadFile("/var/run/ca-trust/tls-ca-bundle.pem")
		if err != nil {
			return fmt.Errorf("unable to read ocm CA trust bundle file: %w", err)
		}
	} else if err != nil && errors.Is(err, os.ErrNotExist) {
		ocmTrustedCABundle, err = ioutil.ReadFile("/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem")
		if err != nil {
			return fmt.Errorf("unable to read ocm CA trust bundle file: %w", err)
		}
	} else {
		return err
	}

	if _, err := trustedCABundle.Write(ocmTrustedCABundle); err != nil {
		return fmt.Errorf("unable to write ocm CA trust bundle to buffer: %w", err)
	}

	if mgmtClusterCaps.Has(capabilities.CapabilityImage) {
		// Adds user-specified image registry CA bundle to the full CA trust bundle if it exists
		imageRegistryCABundle, err := getImageRegistryCABundle(ctx, client)
		if err != nil {
			return fmt.Errorf("unable to get CA bundle for image mirror registries: %w", err)
		}
		if imageRegistryCABundle != nil {
			if _, err := trustedCABundle.Write(imageRegistryCABundle.Bytes()); err != nil {
				return fmt.Errorf("unable to write image registry CA to buffer: %w", err)
			}
		}
	}

	if err := ioutil.WriteFile("/trust-bundle/tls-ca-bundle.pem", trustedCABundle.Bytes(), 0644); err != nil {
		return err
	}

	log.Info("Finished initializing environment for Hypershift Operator")
	return nil
}

// getMgmtClusterCapabilities initializes the ManagementClusterCabilities object
// to determine if the management cluster has the listed capabilities
func getMgmtClusterCapabilities() (*capabilities.ManagementClusterCapabilities, error) {
	config, err := util.GetConfig()
	if err != nil {
		return nil, fmt.Errorf("unable to get kubernetes config: %w", err)
	}
	discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
	if err != nil {
		return nil, fmt.Errorf("unable to get discovery client: %w", err)
	}
	return capabilities.DetectManagementClusterCapabilities(discoveryClient)
}

// getImageRegistryCABundle retrieves the image registry CAs listed under image.config.openshift.io
// and merges them into one CA bundle
func getImageRegistryCABundle(ctx context.Context, client crclient.Client) (*bytes.Buffer, error) {
	img := &configapi.Image{}
	if err := client.Get(ctx, types.NamespacedName{Name: "cluster"}, img); err != nil {
		return nil, err
	}
	if img == nil || img.Spec.AdditionalTrustedCA.Name == "" {
		return nil, nil
	}
	configmap := &corev1.ConfigMap{}
	if err := client.Get(ctx, types.NamespacedName{Name: img.Spec.AdditionalTrustedCA.Name, Namespace: "openshift-config"}, configmap); err != nil {
		return nil, err
	}
	if configmap.Data != nil {
		// Merge all registry CA certificates together into one bundle
		var buf bytes.Buffer
		for _, crt := range configmap.Data {
			buf.WriteString(crt)
		}
		if buf.Len() > 0 {
			return &buf, nil
		}
	}
	return nil, nil
}