Fetch ignition from guest cluster for ignition server

control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go

@@ -900,7 +900,9 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont
 	if err := ignitionserver.ReconcileIgnitionServer(ctx,
 		r.Client,
 		createOrUpdate,
+		releaseImageProvider.Version(),
 		releaseImageProvider.GetImage(util.CPOImageName),
+		releaseImageProvider.GetImage("cli"),
 		hostedControlPlane,
 		r.DefaultIngressDomain,
 		// The healthz handler was added before the CPO started to manage the ignition server, and it's the same binary,

control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go

@@ -2,10 +2,12 @@ package ignitionserver
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"strings"
 
+	configv1 "github.com/openshift/api/config/v1"
 	routev1 "github.com/openshift/api/route/v1"
 	hyperv1 "github.com/openshift/hypershift/api/v1beta1"
 	"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator"
@@ -30,7 +32,9 @@ import (
 func ReconcileIgnitionServer(ctx context.Context,
 	c client.Client,
 	createOrUpdate upsert.CreateOrUpdateFN,
+	releaseVersion string,
 	utilitiesImage string,
+	cliImage string,
 	hcp *hyperv1.HostedControlPlane,
 	defaultIngressDomain string,
 	hasHealthzHandler bool,
@@ -252,6 +256,16 @@ func ReconcileIgnitionServer(ctx context.Context,
 		}
 	}
 
+	if hcp.Status.KubeConfig == nil {
+		return errors.New("kubeconfig is not yet specified in hosted control plane status")
+	}
+
+	// Use the default FeatureSet unless otherwise specified.
+	guestFeatureSet := configv1.Default
+	if hcp.Spec.Configuration != nil && hcp.Spec.Configuration.FeatureGate != nil && hcp.Spec.Configuration.FeatureGate.FeatureSet != "" {
+		guestFeatureSet = hcp.Spec.Configuration.FeatureGate.FeatureSet
+	}
+
 	// Reconcile deployment
 	ignitionServerDeployment := ignitionserver.Deployment(controlPlaneNamespace)
 	if result, err := createOrUpdate(ctx, c, ignitionServerDeployment, func() error {
@@ -295,6 +309,58 @@ func ReconcileIgnitionServer(ctx context.Context,
 								EmptyDir: &corev1.EmptyDirVolumeSource{},
 							},
 						},
+						{
+							Name: "guest-kubeconfig",
+							VolumeSource: corev1.VolumeSource{
+								Secret: &corev1.SecretVolumeSource{
+									// Mount the kubeconfig for the guest cluster to fetch the FeatureGate.
+									SecretName:  hcp.Spec.KubeConfig.Name,
+									DefaultMode: utilpointer.Int32(0640),
+								},
+							},
+						},
+						{
+							Name: "shared",
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
+					},
+					InitContainers: []corev1.Container{
+						{
+							Name:            "fetch-feature-gate",
+							Image:           cliImage,
+							ImagePullPolicy: corev1.PullIfNotPresent,
+							Env: []corev1.EnvVar{
+								{
+									Name:  "KUBECONFIG",
+									Value: fmt.Sprintf("/var/secrets/guest-kubeconfig/%s", hcp.Status.KubeConfig.Key),
+								},
+							},
+							Command: []string{
+								"/bin/bash",
+							},
+							Args: []string{
+								"-c",
+								fetchFeatureGateScript(guestFeatureSet, releaseVersion),
+							},
+							Resources: corev1.ResourceRequirements{
+								Requests: corev1.ResourceList{
+									corev1.ResourceMemory: resource.MustParse("40Mi"),
+									corev1.ResourceCPU:    resource.MustParse("10m"),
+								},
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "guest-kubeconfig",
+									MountPath: "/var/secrets/guest-kubeconfig",
+								},
+								{
+									Name:      "shared",
+									MountPath: "/shared",
+								},
+							},
+						},
 					},
 					Containers: []corev1.Container{
 						{
@@ -318,6 +384,7 @@ func ReconcileIgnitionServer(ctx context.Context,
 								"--key-file", "/var/run/secrets/ignition/serving-cert/tls.key",
 								"--registry-overrides", convertRegistryOverridesToCommandLineFlag(registryOverrides),
 								"--platform", string(hcp.Spec.Platform.Type),
+								"--feature-gate=/shared/99_feature-gate.yaml",
 							},
 							LivenessProbe: &corev1.Probe{
 								ProbeHandler:        probeHandler,
@@ -360,6 +427,10 @@ func ReconcileIgnitionServer(ctx context.Context,
 									Name:      "payloads",
 									MountPath: "/payloads",
 								},
+								{
+									Name:      "shared",
+									MountPath: "/shared",
+								},
 							},
 						},
 					},
@@ -467,3 +538,22 @@ func convertRegistryOverridesToCommandLineFlag(registryOverrides map[string]stri
 	// this is the equivalent of null on a StringToString command line variable.
 	return "="
 }
+
+func fetchFeatureGateScript(featureSet configv1.FeatureSet, payloadVersion string) string {
+	var script = `#!/bin/sh
+if [[ $(oc get featuregate cluster -o json | jq -r .spec.featureSet) != %s ]]; then
+	echo "FeatureGate has not been updated with current feature set yet."
+	sleep 1
+	exit 1
+fi
+
+if [[ ! -z $(oc get featuregate cluster -o json | jq '.status.featureGates[] | select(.version == "%s")') ]]; then
+	echo "FeatureGate has not rendered current payload version yet."
+	sleep 1
+	exit 1
+fi
+
+oc get featuregate cluster -o yaml > /shared/99_feature-gate.yaml
+`
+	return fmt.Sprintf(script, featureSet, payloadVersion)
+}

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -1539,12 +1539,25 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques
 		return ctrl.Result{}, fmt.Errorf("failed to reconcile control plane operator: %w", err)
 	}
 
+	cliImage, releaseVersion, err := func() (string,string, error) {
+		releaseInfo, err := r.ReleaseProvider.Lookup(ctx, hcluster.Spec.Release.Image, pullSecretBytes)
+		if err != nil {
+			return "", "", fmt.Errorf("failed to lookup release image: %w", err)
+		}
+		return releaseInfo.ComponentImages()["cli"], releaseInfo.Version(), nil
+	}()
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("failed to get cli image: %w", err)
+	}
+
 	// Reconcile the Ignition server
 	if !controlplaneOperatorManagesIgnitionServer {
 		if err := ignitionserverreconciliation.ReconcileIgnitionServer(ctx,
 			r.Client,
 			createOrUpdate,
+			releaseVersion,
 			utilitiesImage,
+			cliImage,
 			hcp,
 			defaultIngressDomain,
 			ignitionServerHasHealthzHandler,

ignition-server/cmd/run_local_ignitionprovider.go

@@ -25,6 +25,7 @@ type RunLocalIgnitionProviderOptions struct {
 	Image       string
 	TokenSecret string
 	WorkDir     string
+	FeatureGate string
 }
 
 func NewRunLocalIgnitionProviderCommand() *cobra.Command {
@@ -39,6 +40,7 @@ func NewRunLocalIgnitionProviderCommand() *cobra.Command {
 	cmd.Flags().StringVar(&opts.Image, "image", opts.Image, "Release image")
 	cmd.Flags().StringVar(&opts.TokenSecret, "token-secret", opts.TokenSecret, "Token secret name")
 	cmd.Flags().StringVar(&opts.WorkDir, "dir", opts.WorkDir, "Working directory (default: temporary dir)")
+	cmd.Flags().StringVar(&opts.FeatureGate, "feature-gate", opts.FeatureGate, "Path to a rendered featuregates.config.openshift.io/v1 file")
 
 	cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		ctx, cancel := context.WithCancel(context.Background())
@@ -101,6 +103,7 @@ func (o *RunLocalIgnitionProviderOptions) Run(ctx context.Context) error {
 		WorkDir:         o.WorkDir,
 		PreserveOutput:  true,
 		ImageFileCache:  imageFileCache,
+		FeatureGate:     o.FeatureGate,
 	}
 
 	payload, err := p.GetPayload(ctx, o.Image, config.String(), "")

ignition-server/cmd/start.go

@@ -56,6 +56,7 @@ type Options struct {
 	Platform          string
 	WorkDir           string
 	MetricsAddr       string
+	FeatureGate       string
 }
 
 // This is a https server that enable us to satisfy
@@ -88,6 +89,7 @@ func NewStartCommand() *cobra.Command {
 	cmd.Flags().StringVar(&opts.Platform, "platform", "", "The cloud provider platform name")
 	cmd.Flags().StringVar(&opts.WorkDir, "work-dir", opts.WorkDir, "Directory in which to store transient working data")
 	cmd.Flags().StringVar(&opts.MetricsAddr, "metrics-addr", opts.MetricsAddr, "The address the metric endpoint binds to.")
+	cmd.Flags().StringVar(&opts.FeatureGate, "feature-gate", opts.FeatureGate, "Path to a rendered featuregates.config.openshift.io/v1 file")
 
 	cmd.Run = func(cmd *cobra.Command, args []string) {
 		ctx, cancel := context.WithCancel(context.Background())
@@ -110,7 +112,7 @@ func NewStartCommand() *cobra.Command {
 
 // setUpPayloadStoreReconciler sets up manager with a TokenSecretReconciler controller
 // to keep the PayloadStore up to date.
-func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[string]string, cloudProvider hyperv1.PlatformType, cacheDir string, metricsAddr string) (ctrl.Manager, error) {
+func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[string]string, cloudProvider hyperv1.PlatformType, cacheDir string, metricsAddr string, featureGate string) (ctrl.Manager, error) {
 	if os.Getenv(namespaceEnvVariableName) == "" {
 		return nil, fmt.Errorf("environment variable %s is empty, this is not supported", namespaceEnvVariableName)
 	}
@@ -155,6 +157,7 @@ func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[stri
 			CloudProvider:  cloudProvider,
 			WorkDir:        cacheDir,
 			ImageFileCache: imageFileCache,
+			FeatureGate:    featureGate,
 		},
 	}).SetupWithManager(ctx, mgr); err != nil {
 		return nil, fmt.Errorf("unable to create controller: %w", err)
@@ -175,7 +178,7 @@ func run(ctx context.Context, opts Options) error {
 		return fmt.Errorf("failed to load serving cert: %w", err)
 	}
 
-	mgr, err := setUpPayloadStoreReconciler(ctx, opts.RegistryOverrides, hyperv1.PlatformType(opts.Platform), opts.WorkDir, opts.MetricsAddr)
+	mgr, err := setUpPayloadStoreReconciler(ctx, opts.RegistryOverrides, hyperv1.PlatformType(opts.Platform), opts.WorkDir, opts.MetricsAddr, opts.FeatureGate)
 	if err != nil {
 		return fmt.Errorf("error setting up manager: %w", err)
 	}

ignition-server/controllers/local_ignitionprovider.go

@@ -18,7 +18,6 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/docker/distribution/manifest/manifestlist"
-	configv1 "github.com/openshift/api/config/v1"
 	hyperv1 "github.com/openshift/hypershift/api/v1beta1"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common"
 	"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider"
@@ -67,6 +66,11 @@ type LocalIgnitionProvider struct {
 	// deleted after use.
 	PreserveOutput bool
 
+	// FeatureGate is the path to a rendered feature gate manifest.
+	// This must be copied into the MCC directory as it is required
+	// to render the ignition payload.
+	FeatureGate string
+
 	ImageFileCache *imageFileCache
 
 	lock sync.Mutex
@@ -237,15 +241,11 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str
 	}
 
 	// Write out the feature gate manifest to the MCC dir.
+	// Use the feature gate from the hosted control plane which should reflect the feature gate of the cluster.
 	if err := func() error {
-		featureGate := &configv1.FeatureGate{}
-		if err := p.Client.Get(ctx, client.ObjectKey{Name: "cluster"}, featureGate); err != nil {
-			return fmt.Errorf("failed to get feature gate: %w", err)
-		}
-
-		featureGateBytes, err := yaml.Marshal(featureGate)
+		featureGateBytes, err := os.ReadFile(p.FeatureGate)
 		if err != nil {
-			return fmt.Errorf("failed to marshal feature gate: %w", err)
+			return fmt.Errorf("failed to read feature gate: %w", err)
 		}
 
 		if err := os.WriteFile(filepath.Join(mccBaseDir, "99_feature-gate.yaml"), featureGateBytes, 0644); err != nil {