Initialize RegistryOverrides w/ mgmt cluster ICSP

Initialize RegistryOverrides with the management cluster's ICSP if no
value was passed in for the RegistryOverrides flag.

Signed-off-by: Bryan Cox <brcox@redhat.com>

api/scheme.go

@@ -6,7 +6,7 @@ import (
 	snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
 	configv1 "github.com/openshift/api/config/v1"
 	imagev1 "github.com/openshift/api/image/v1"
-	operatorv1 "github.com/openshift/api/operator/v1"
+	operatorv1 "github.com/openshift/api/operator/v1alpha1"
 	routev1 "github.com/openshift/api/route/v1"
 	securityv1 "github.com/openshift/api/security/v1"
 	agentv1 "github.com/openshift/cluster-api-provider-agent/api/v1alpha1"

control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go

@@ -3,9 +3,6 @@ package ignitionserver
 import (
 	"context"
 	"fmt"
-	"net"
-	"strings"
-
 	routev1 "github.com/openshift/api/route/v1"
 	hyperv1 "github.com/openshift/hypershift/api/v1beta1"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator"
@@ -23,6 +20,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	utilpointer "k8s.io/utils/pointer"
+	"net"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -34,7 +32,7 @@ func ReconcileIgnitionServer(ctx context.Context,
 	hcp *hyperv1.HostedControlPlane,
 	defaultIngressDomain string,
 	hasHealthzHandler bool,
-	registryOverrides map[string]string,
+	registryOverrides map[string][]string,
 	managementClusterHasCapabilitySecurityContextConstraint bool,
 	ownerRef config.OwnerRef,
 ) error {
@@ -310,7 +308,7 @@ func ReconcileIgnitionServer(ctx context.Context,
 								"ignition-server",
 								"--cert-file", "/var/run/secrets/ignition/serving-cert/tls.crt",
 								"--key-file", "/var/run/secrets/ignition/serving-cert/tls.key",
-								"--registry-overrides", convertRegistryOverridesToCommandLineFlag(registryOverrides),
+								"--registry-overrides", util.ConvertRegistryOverridesToCommandLineFlag(registryOverrides),
 								"--platform", string(hcp.Spec.Platform.Type),
 							},
 							LivenessProbe: &corev1.Probe{
@@ -449,15 +447,3 @@ func reconcileInternalRoute(route *routev1.Route, ownerRef config.OwnerRef) erro
 	// Assumes ownerRef is the HCP
 	return util.ReconcileInternalRoute(route, ownerRef.Reference.Name, ignitionserver.Service(route.Namespace).Name)
 }
-
-func convertRegistryOverridesToCommandLineFlag(registryOverrides map[string]string) string {
-	commandLineFlagArray := []string{}
-	for registrySource, registryReplacement := range registryOverrides {
-		commandLineFlagArray = append(commandLineFlagArray, fmt.Sprintf("%s=%s", registrySource, registryReplacement))
-	}
-	if len(commandLineFlagArray) > 0 {
-		return strings.Join(commandLineFlagArray, ",")
-	}
-	// this is the equivalent of null on a StringToString command line variable.
-	return "="
-}

control-plane-operator/main.go

@@ -350,7 +350,7 @@ func NewStartCommand() *cobra.Command {
 				},
 				ComponentImages: componentImages,
 			},
-			RegistryOverrides: registryOverrides,
+			RegistryOverrides: util.ConvertRegistryOverrideFlagDataType(registryOverrides),
 		}
 
 		defaultIngressDomain := os.Getenv(config.DefaultIngressDomainEnvVar)

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -1903,7 +1903,7 @@ func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Cont
 	// Reconcile operator deployment
 	controlPlaneOperatorDeployment := controlplaneoperator.OperatorDeployment(controlPlaneNamespace.Name)
 	_, err = createOrUpdate(ctx, r.Client, controlPlaneOperatorDeployment, func() error {
-		return reconcileControlPlaneOperatorDeployment(controlPlaneOperatorDeployment, hcluster, hostedControlPlane, controlPlaneOperatorImage, utilitiesImage, r.SetDefaultSecurityContext, controlPlaneOperatorServiceAccount, r.EnableCIDebugOutput, convertRegistryOverridesToCommandLineFlag(r.ReleaseProvider.GetRegistryOverrides()), defaultIngressDomain, cpoHasUtilities, r.MetricsSet)
+		return reconcileControlPlaneOperatorDeployment(controlPlaneOperatorDeployment, hcluster, hostedControlPlane, controlPlaneOperatorImage, utilitiesImage, r.SetDefaultSecurityContext, controlPlaneOperatorServiceAccount, r.EnableCIDebugOutput, hyperutil.ConvertRegistryOverridesToCommandLineFlag(r.ReleaseProvider.GetRegistryOverrides()), defaultIngressDomain, cpoHasUtilities, r.MetricsSet)
 	})
 	if err != nil {
 		return fmt.Errorf("failed to reconcile controlplane operator deployment: %w", err)
@@ -1937,18 +1937,6 @@ func (r *HostedClusterReconciler) reconcileControlPlaneOperator(ctx context.Cont
 	return nil
 }
 
-func convertRegistryOverridesToCommandLineFlag(registryOverrides map[string]string) string {
-	commandLineFlagArray := []string{}
-	for registrySource, registryReplacement := range registryOverrides {
-		commandLineFlagArray = append(commandLineFlagArray, fmt.Sprintf("%s=%s", registrySource, registryReplacement))
-	}
-	if len(commandLineFlagArray) > 0 {
-		return strings.Join(commandLineFlagArray, ",")
-	}
-	// this is the equivalent of null on a StringToString command line variable.
-	return "="
-}
-
 func servicePublishingStrategyByType(hcp *hyperv1.HostedCluster, svcType hyperv1.ServiceType) *hyperv1.ServicePublishingStrategy {
 	for _, mapping := range hcp.Spec.Services {
 		if mapping.Service == svcType {

hypershift-operator/main.go

@@ -19,6 +19,8 @@ package main
 import (
 	"context"
 	"fmt"
+	cmdutil "github.com/openshift/hypershift/cmd/util"
+	"github.com/openshift/hypershift/support/globalconfig"
 	"os"
 	"strings"
 	"time"
@@ -228,6 +230,21 @@ func run(ctx context.Context, opts *StartOptions, log logr.Logger) error {
 	}
 	log.Info("Using metrics set", "set", metricsSet.String())
 
+	// Convert RegistryOverrides flag to a map of string-to-slice of strings
+	registryOverrides := util.ConvertRegistryOverrideFlagDataType(opts.RegistryOverrides)
+
+	// Populate RegistryOverrides with ICSP from management cluster if no value is passed in the RegistryOverrides flag
+	if len(registryOverrides) == 0 {
+		client, err := cmdutil.GetClient()
+		if err != nil {
+			return err
+		}
+		registryOverrides, err = populateRegistryOverrides(ctx, client)
+		if err != nil {
+			return err
+		}
+	}
+
 	hostedClusterReconciler := &hostedcluster.HostedClusterReconciler{
 		Client:                        mgr.GetClient(),
 		ManagementClusterCapabilities: mgmtClusterCaps,
@@ -237,7 +254,7 @@ func run(ctx context.Context, opts *StartOptions, log logr.Logger) error {
 				Inner: &releaseinfo.RegistryClientProvider{},
 				Cache: map[string]*releaseinfo.ReleaseImage{},
 			},
-			RegistryOverrides: opts.RegistryOverrides,
+			RegistryOverrides: registryOverrides,
 		},
 		EnableOCPClusterMonitoring: opts.EnableOCPClusterMonitoring,
 		EnableCIDebugOutput:        opts.EnableCIDebugOutput,
@@ -286,7 +303,7 @@ func run(ctx context.Context, opts *StartOptions, log logr.Logger) error {
 				Inner: &releaseinfo.RegistryClientProvider{},
 				Cache: map[string]*releaseinfo.ReleaseImage{},
 			},
-			RegistryOverrides: opts.RegistryOverrides,
+			RegistryOverrides: registryOverrides,
 		},
 		CreateOrUpdateProvider:  createOrUpdate,
 		HypershiftOperatorImage: operatorImage,
@@ -381,3 +398,27 @@ func run(ctx context.Context, opts *StartOptions, log logr.Logger) error {
 	log.Info("starting manager")
 	return mgr.Start(ctx)
 }
+
+// populateRegistryOverrides returns a map of registry overrides from the management cluster
+func populateRegistryOverrides(ctx context.Context, client crclient.Client) (map[string][]string, error) {
+	var icspMapping = make(map[string][]string)
+	var listOfICSP = globalconfig.ImageContentSourcePolicyList()
+
+	err := client.List(ctx, listOfICSP)
+	if err != nil {
+		return nil, err
+	}
+
+	// For each ICSP, map the source with each of its mirrors
+	for _, item := range listOfICSP.Items {
+		for _, registryDigestMirror := range item.Spec.RepositoryDigestMirrors {
+			source := registryDigestMirror.Source
+
+			for n := range registryDigestMirror.Mirrors {
+				icspMapping[source] = append(icspMapping[source], registryDigestMirror.Mirrors[n])
+			}
+		}
+	}
+
+	return icspMapping, nil
+}

ignition-server/cmd/start.go

@@ -110,7 +110,7 @@ func NewStartCommand() *cobra.Command {
 
 // setUpPayloadStoreReconciler sets up manager with a TokenSecretReconciler controller
 // to keep the PayloadStore up to date.
-func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[string]string, cloudProvider hyperv1.PlatformType, cacheDir string, metricsAddr string) (ctrl.Manager, error) {
+func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[string][]string, cloudProvider hyperv1.PlatformType, cacheDir string, metricsAddr string) (ctrl.Manager, error) {
 	if os.Getenv(namespaceEnvVariableName) == "" {
 		return nil, fmt.Errorf("environment variable %s is empty, this is not supported", namespaceEnvVariableName)
 	}
@@ -175,7 +175,10 @@ func run(ctx context.Context, opts Options) error {
 		return fmt.Errorf("failed to load serving cert: %w", err)
 	}
 
-	mgr, err := setUpPayloadStoreReconciler(ctx, opts.RegistryOverrides, hyperv1.PlatformType(opts.Platform), opts.WorkDir, opts.MetricsAddr)
+	// Convert RegistryOverrides flag to a map of string-to-slice of strings
+	registryOverrides := util.ConvertRegistryOverrideFlagDataType(opts.RegistryOverrides)
+
+	mgr, err := setUpPayloadStoreReconciler(ctx, registryOverrides, hyperv1.PlatformType(opts.Platform), opts.WorkDir, opts.MetricsAddr)
 	if err != nil {
 		return fmt.Errorf("error setting up manager: %w", err)
 	}

support/globalconfig/imagecontentsource.go

@@ -19,6 +19,15 @@ func ImageContentSourcePolicy() *operatorv1alpha1.ImageContentSourcePolicy {
 	}
 }
 
+func ImageContentSourcePolicyList() *operatorv1alpha1.ImageContentSourcePolicyList {
+	return &operatorv1alpha1.ImageContentSourcePolicyList{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "ImageContentSourcePolicyList",
+			APIVersion: operatorv1alpha1.GroupVersion.String(),
+		},
+	}
+}
+
 func ReconcileImageContentSourcePolicy(icsp *operatorv1alpha1.ImageContentSourcePolicy, hcp *hyperv1.HostedControlPlane) error {
 	if icsp.Labels == nil {
 		icsp.Labels = map[string]string{}

support/releaseinfo/fake/fake.go

@@ -66,6 +66,6 @@ func (f *FakeReleaseProvider) Lookup(ctx context.Context, image string, pullSecr
 	return releaseImage, nil
 }
 
-func (*FakeReleaseProvider) GetRegistryOverrides() map[string]string {
+func (*FakeReleaseProvider) GetRegistryOverrides() map[string][]string {
 	return nil
 }

support/releaseinfo/registry_mirror_provider.go

@@ -18,7 +18,7 @@ type RegistryMirrorProviderDecorator struct {
 	// images before being applied are scanned for the source registry string and if found the string is replaced with
 	// the destination registry string. This allows hypershift to run in non-crio environments where mirroring is not
 	// applicable.
-	RegistryOverrides map[string]string
+	RegistryOverrides map[string][]string
 
 	lock sync.Mutex
 }
@@ -33,12 +33,14 @@ func (p *RegistryMirrorProviderDecorator) Lookup(ctx context.Context, image stri
 	}
 	for i := range releaseImage.ImageStream.Spec.Tags {
 		for registrySource, registryDest := range p.RegistryOverrides {
-			releaseImage.ImageStream.Spec.Tags[i].From.Name = strings.Replace(releaseImage.ImageStream.Spec.Tags[i].From.Name, registrySource, registryDest, 1)
+			for _, registryReplacement := range registryDest {
+				releaseImage.ImageStream.Spec.Tags[i].From.Name = strings.Replace(releaseImage.ImageStream.Spec.Tags[i].From.Name, registrySource, registryReplacement, 1)
+			}
 		}
 	}
 	return releaseImage, nil
 }
 
-func (p *RegistryMirrorProviderDecorator) GetRegistryOverrides() map[string]string {
+func (p *RegistryMirrorProviderDecorator) GetRegistryOverrides() map[string][]string {
 	return p.RegistryOverrides
 }

support/releaseinfo/releaseinfo.go

@@ -22,7 +22,7 @@ type Provider interface {
 
 type ProviderWithRegistryOverrides interface {
 	Provider
-	GetRegistryOverrides() map[string]string
+	GetRegistryOverrides() map[string][]string
 }
 
 // ReleaseImage wraps an ImageStream with some utilities that help the user

support/util/util.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 
@@ -188,3 +189,28 @@ func HashStruct(o interface{}) string {
 	intHash := hash.Sum32()
 	return fmt.Sprintf("%08x", intHash)
 }
+
+// ConvertRegistryOverridesToCommandLineFlag converts a map of registry sources and their mirrors into a string that can be ingested as a command line variable
+func ConvertRegistryOverridesToCommandLineFlag(registryOverrides map[string][]string) string {
+	var commandLineFlagArray []string
+	for registrySource, registryReplacements := range registryOverrides {
+		for _, registryReplacement := range registryReplacements {
+			commandLineFlagArray = append(commandLineFlagArray, fmt.Sprintf("%s=%s", registrySource, registryReplacement))
+		}
+	}
+	if len(commandLineFlagArray) > 0 {
+		sort.Strings(commandLineFlagArray)
+		return strings.Join(commandLineFlagArray, ",")
+	}
+	// this is the equivalent of null on a StringToString command line variable.
+	return "="
+}
+
+// ConvertRegistryOverrideFlagDataType converts RegistryOverride data type from map[string]string to a map[string][]string
+func ConvertRegistryOverrideFlagDataType(registryOverrides map[string]string) map[string][]string {
+	registryOverrideSliceMap := make(map[string][]string)
+	for source, mirror := range registryOverrides {
+		registryOverrideSliceMap[source] = append(registryOverrideSliceMap[source], mirror)
+	}
+	return registryOverrideSliceMap
+}

support/util/util_test.go

@@ -49,6 +49,102 @@ func TestCompressDecompress(t *testing.T) {
 	}
 }
 
+func TestConvertRegistryOverridesToCommandLineFlag(t *testing.T) {
+	testCases := []struct {
+		name              string
+		registryOverrides map[string][]string
+		expectedFlag      string
+	}{
+		{
+			name:         "No registry overrides",
+			expectedFlag: "=",
+		},
+		{
+			name: "Registry overrides with single mirrors",
+			registryOverrides: map[string][]string{
+				"registry1": {
+					"mirror1.1",
+				},
+				"registry2": {
+					"mirror2.1",
+				},
+				"registry3": {
+					"mirror3.1",
+				},
+			},
+			expectedFlag: "registry1=mirror1.1,registry2=mirror2.1,registry3=mirror3.1",
+		},
+		{
+			name: "Registry overrides with multiple mirrors",
+			registryOverrides: map[string][]string{
+				"registry1": {
+					"mirror1.1",
+					"mirror1.2",
+					"mirror1.3",
+				},
+				"registry2": {
+					"mirror2.1",
+					"mirror2.2",
+				},
+				"registry3": {
+					"mirror3.1",
+				},
+			},
+			expectedFlag: "registry1=mirror1.1,registry1=mirror1.2,registry1=mirror1.3,registry2=mirror2.1,registry2=mirror2.2,registry3=mirror3.1",
+		},
+	}
+
+	t.Parallel()
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			g := NewGomegaWithT(t)
+
+			result := ConvertRegistryOverridesToCommandLineFlag(testCase.registryOverrides)
+			g.Expect(result).To(Equal(testCase.expectedFlag))
+		})
+	}
+}
+
+func TestConvertRegistryOverrideFlagDataType(t *testing.T) {
+	testCases := []struct {
+		name                 string
+		registryOverrideFlag map[string]string
+		expectedResult       map[string][]string
+	}{
+		{
+			name:           "No registry overrides",
+			expectedResult: map[string][]string{},
+		},
+		{
+			name: "Registry overrides",
+			registryOverrideFlag: map[string]string{
+				"registry1": "mirror1",
+				"registry2": "mirror2",
+			},
+			expectedResult: map[string][]string{
+				"registry1": {
+					"mirror1",
+				},
+				"registry2": {
+					"mirror2",
+				},
+			},
+		},
+	}
+
+	t.Parallel()
+
+	for _, testCase := range testCases {
+		t.Run(testCase.name, func(t *testing.T) {
+			g := NewGomegaWithT(t)
+
+			result := ConvertRegistryOverrideFlagDataType(testCase.registryOverrideFlag)
+			g.Expect(result).To(Equal(testCase.expectedResult))
+		})
+	}
+}
+
 // Tests that a given input can be expected and encoded without errors.
 func testCompressFunc(t *testing.T, payload, expected []byte) {
 	t.Helper()