From a60b38c8f9084968adf6a6f367f0afaf060f6931 Mon Sep 17 00:00:00 2001 From: Joel Speed Date: Mon, 22 May 2023 18:47:59 +0100 Subject: [PATCH] Fetch ignition from guest cluster for ignition server --- .../hostedcontrolplane_controller.go | 2 + .../ignitionserver/ignitionserver.go | 90 +++++++++++++++++++ .../hostedcluster/hostedcluster_controller.go | 13 +++ .../cmd/run_local_ignitionprovider.go | 3 + ignition-server/cmd/start.go | 7 +- .../controllers/local_ignitionprovider.go | 16 ++-- 6 files changed, 121 insertions(+), 10 deletions(-) diff --git a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go index 7ca22b7437e..fa05f930f56 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go +++ b/control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go @@ -900,7 +900,9 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont if err := ignitionserver.ReconcileIgnitionServer(ctx, r.Client, createOrUpdate, + releaseImageProvider.Version(), releaseImageProvider.GetImage(util.CPOImageName), + releaseImageProvider.GetImage("cli"), hostedControlPlane, r.DefaultIngressDomain, // The healthz handler was added before the CPO started to manage the ignition server, and it's the same binary, diff --git a/control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go b/control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go index 52205c74807..d84b823d7c4 100644 --- a/control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go +++ b/control-plane-operator/controllers/hostedcontrolplane/ignitionserver/ignitionserver.go @@ -2,10 +2,12 @@ package ignitionserver import ( "context" + "errors" "fmt" "net" "strings" + configv1 "github.com/openshift/api/config/v1" routev1 "github.com/openshift/api/route/v1" hyperv1 "github.com/openshift/hypershift/api/v1beta1" "github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator" @@ -30,7 +32,9 @@ import ( func ReconcileIgnitionServer(ctx context.Context, c client.Client, createOrUpdate upsert.CreateOrUpdateFN, + releaseVersion string, utilitiesImage string, + cliImage string, hcp *hyperv1.HostedControlPlane, defaultIngressDomain string, hasHealthzHandler bool, @@ -252,6 +256,16 @@ func ReconcileIgnitionServer(ctx context.Context, } } + if hcp.Status.KubeConfig == nil { + return errors.New("kubeconfig is not yet specified in hosted control plane status") + } + + // Use the default FeatureSet unless otherwise specified. + guestFeatureSet := configv1.Default + if hcp.Spec.Configuration != nil && hcp.Spec.Configuration.FeatureGate != nil && hcp.Spec.Configuration.FeatureGate.FeatureSet != "" { + guestFeatureSet = hcp.Spec.Configuration.FeatureGate.FeatureSet + } + // Reconcile deployment ignitionServerDeployment := ignitionserver.Deployment(controlPlaneNamespace) if result, err := createOrUpdate(ctx, c, ignitionServerDeployment, func() error { @@ -295,6 +309,58 @@ func ReconcileIgnitionServer(ctx context.Context, EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, + { + Name: "guest-kubeconfig", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + // Mount the kubeconfig for the guest cluster to fetch the FeatureGate. + SecretName: hcp.Spec.KubeConfig.Name, + DefaultMode: utilpointer.Int32(0640), + }, + }, + }, + { + Name: "shared", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + InitContainers: []corev1.Container{ + { + Name: "fetch-feature-gate", + Image: cliImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Env: []corev1.EnvVar{ + { + Name: "KUBECONFIG", + Value: fmt.Sprintf("/var/secrets/guest-kubeconfig/%s", hcp.Status.KubeConfig.Key), + }, + }, + Command: []string{ + "/bin/bash", + }, + Args: []string{ + "-c", + fetchFeatureGateScript(guestFeatureSet, releaseVersion), + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("40Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "guest-kubeconfig", + MountPath: "/var/secrets/guest-kubeconfig", + }, + { + Name: "shared", + MountPath: "/shared", + }, + }, + }, }, Containers: []corev1.Container{ { @@ -318,6 +384,7 @@ func ReconcileIgnitionServer(ctx context.Context, "--key-file", "/var/run/secrets/ignition/serving-cert/tls.key", "--registry-overrides", convertRegistryOverridesToCommandLineFlag(registryOverrides), "--platform", string(hcp.Spec.Platform.Type), + "--feature-gate=/shared/99_feature-gate.yaml", }, LivenessProbe: &corev1.Probe{ ProbeHandler: probeHandler, @@ -360,6 +427,10 @@ func ReconcileIgnitionServer(ctx context.Context, Name: "payloads", MountPath: "/payloads", }, + { + Name: "shared", + MountPath: "/shared", + }, }, }, }, @@ -467,3 +538,22 @@ func convertRegistryOverridesToCommandLineFlag(registryOverrides map[string]stri // this is the equivalent of null on a StringToString command line variable. return "=" } + +func fetchFeatureGateScript(featureSet configv1.FeatureSet, payloadVersion string) string { + var script = `#!/bin/sh +if [[ $(oc get featuregate cluster -o json | jq -r .spec.featureSet) != %s ]]; then + echo "FeatureGate has not been updated with current feature set yet." + sleep 1 + exit 1 +fi + +if [[ ! -z $(oc get featuregate cluster -o json | jq '.status.featureGates[] | select(.version == "%s")') ]]; then + echo "FeatureGate has not rendered current payload version yet." + sleep 1 + exit 1 +fi + +oc get featuregate cluster -o yaml > /shared/99_feature-gate.yaml +` + return fmt.Sprintf(script, featureSet, payloadVersion) +} diff --git a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go index b4c5ff1d8dd..b612c2a5727 100644 --- a/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go +++ b/hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go @@ -1539,12 +1539,25 @@ func (r *HostedClusterReconciler) reconcile(ctx context.Context, req ctrl.Reques return ctrl.Result{}, fmt.Errorf("failed to reconcile control plane operator: %w", err) } + cliImage, releaseVersion, err := func() (string,string, error) { + releaseInfo, err := r.ReleaseProvider.Lookup(ctx, hcluster.Spec.Release.Image, pullSecretBytes) + if err != nil { + return "", "", fmt.Errorf("failed to lookup release image: %w", err) + } + return releaseInfo.ComponentImages()["cli"], releaseInfo.Version(), nil + }() + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get cli image: %w", err) + } + // Reconcile the Ignition server if !controlplaneOperatorManagesIgnitionServer { if err := ignitionserverreconciliation.ReconcileIgnitionServer(ctx, r.Client, createOrUpdate, + releaseVersion, utilitiesImage, + cliImage, hcp, defaultIngressDomain, ignitionServerHasHealthzHandler, diff --git a/ignition-server/cmd/run_local_ignitionprovider.go b/ignition-server/cmd/run_local_ignitionprovider.go index 9dd74488d05..3da67288939 100644 --- a/ignition-server/cmd/run_local_ignitionprovider.go +++ b/ignition-server/cmd/run_local_ignitionprovider.go @@ -25,6 +25,7 @@ type RunLocalIgnitionProviderOptions struct { Image string TokenSecret string WorkDir string + FeatureGate string } func NewRunLocalIgnitionProviderCommand() *cobra.Command { @@ -39,6 +40,7 @@ func NewRunLocalIgnitionProviderCommand() *cobra.Command { cmd.Flags().StringVar(&opts.Image, "image", opts.Image, "Release image") cmd.Flags().StringVar(&opts.TokenSecret, "token-secret", opts.TokenSecret, "Token secret name") cmd.Flags().StringVar(&opts.WorkDir, "dir", opts.WorkDir, "Working directory (default: temporary dir)") + cmd.Flags().StringVar(&opts.FeatureGate, "feature-gate", opts.FeatureGate, "Path to a rendered featuregates.config.openshift.io/v1 file") cmd.RunE = func(cmd *cobra.Command, args []string) error { ctx, cancel := context.WithCancel(context.Background()) @@ -101,6 +103,7 @@ func (o *RunLocalIgnitionProviderOptions) Run(ctx context.Context) error { WorkDir: o.WorkDir, PreserveOutput: true, ImageFileCache: imageFileCache, + FeatureGate: o.FeatureGate, } payload, err := p.GetPayload(ctx, o.Image, config.String(), "") diff --git a/ignition-server/cmd/start.go b/ignition-server/cmd/start.go index aa1a615be10..1257b0b0f2a 100644 --- a/ignition-server/cmd/start.go +++ b/ignition-server/cmd/start.go @@ -56,6 +56,7 @@ type Options struct { Platform string WorkDir string MetricsAddr string + FeatureGate string } // This is a https server that enable us to satisfy @@ -88,6 +89,7 @@ func NewStartCommand() *cobra.Command { cmd.Flags().StringVar(&opts.Platform, "platform", "", "The cloud provider platform name") cmd.Flags().StringVar(&opts.WorkDir, "work-dir", opts.WorkDir, "Directory in which to store transient working data") cmd.Flags().StringVar(&opts.MetricsAddr, "metrics-addr", opts.MetricsAddr, "The address the metric endpoint binds to.") + cmd.Flags().StringVar(&opts.FeatureGate, "feature-gate", opts.FeatureGate, "Path to a rendered featuregates.config.openshift.io/v1 file") cmd.Run = func(cmd *cobra.Command, args []string) { ctx, cancel := context.WithCancel(context.Background()) @@ -110,7 +112,7 @@ func NewStartCommand() *cobra.Command { // setUpPayloadStoreReconciler sets up manager with a TokenSecretReconciler controller // to keep the PayloadStore up to date. -func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[string]string, cloudProvider hyperv1.PlatformType, cacheDir string, metricsAddr string) (ctrl.Manager, error) { +func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[string]string, cloudProvider hyperv1.PlatformType, cacheDir string, metricsAddr string, featureGate string) (ctrl.Manager, error) { if os.Getenv(namespaceEnvVariableName) == "" { return nil, fmt.Errorf("environment variable %s is empty, this is not supported", namespaceEnvVariableName) } @@ -155,6 +157,7 @@ func setUpPayloadStoreReconciler(ctx context.Context, registryOverrides map[stri CloudProvider: cloudProvider, WorkDir: cacheDir, ImageFileCache: imageFileCache, + FeatureGate: featureGate, }, }).SetupWithManager(ctx, mgr); err != nil { return nil, fmt.Errorf("unable to create controller: %w", err) @@ -175,7 +178,7 @@ func run(ctx context.Context, opts Options) error { return fmt.Errorf("failed to load serving cert: %w", err) } - mgr, err := setUpPayloadStoreReconciler(ctx, opts.RegistryOverrides, hyperv1.PlatformType(opts.Platform), opts.WorkDir, opts.MetricsAddr) + mgr, err := setUpPayloadStoreReconciler(ctx, opts.RegistryOverrides, hyperv1.PlatformType(opts.Platform), opts.WorkDir, opts.MetricsAddr, opts.FeatureGate) if err != nil { return fmt.Errorf("error setting up manager: %w", err) } diff --git a/ignition-server/controllers/local_ignitionprovider.go b/ignition-server/controllers/local_ignitionprovider.go index 33222564a52..87d2874bb8e 100644 --- a/ignition-server/controllers/local_ignitionprovider.go +++ b/ignition-server/controllers/local_ignitionprovider.go @@ -18,7 +18,6 @@ import ( "github.com/blang/semver" "github.com/docker/distribution/manifest/manifestlist" - configv1 "github.com/openshift/api/config/v1" hyperv1 "github.com/openshift/hypershift/api/v1beta1" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/common" "github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/imageprovider" @@ -67,6 +66,11 @@ type LocalIgnitionProvider struct { // deleted after use. PreserveOutput bool + // FeatureGate is the path to a rendered feature gate manifest. + // This must be copied into the MCC directory as it is required + // to render the ignition payload. + FeatureGate string + ImageFileCache *imageFileCache lock sync.Mutex @@ -237,15 +241,11 @@ func (p *LocalIgnitionProvider) GetPayload(ctx context.Context, releaseImage str } // Write out the feature gate manifest to the MCC dir. + // Use the feature gate from the hosted control plane which should reflect the feature gate of the cluster. if err := func() error { - featureGate := &configv1.FeatureGate{} - if err := p.Client.Get(ctx, client.ObjectKey{Name: "cluster"}, featureGate); err != nil { - return fmt.Errorf("failed to get feature gate: %w", err) - } - - featureGateBytes, err := yaml.Marshal(featureGate) + featureGateBytes, err := os.ReadFile(p.FeatureGate) if err != nil { - return fmt.Errorf("failed to marshal feature gate: %w", err) + return fmt.Errorf("failed to read feature gate: %w", err) } if err := os.WriteFile(filepath.Join(mccBaseDir, "99_feature-gate.yaml"), featureGateBytes, 0644); err != nil {