Skip to content

ARO-14354,ARO-14356,ARO-14355: introduce image registry capability behind feature gate #5456

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
openshift-merge-bot merged 1 commit into from
Mar 12, 2025
Merged

ARO-14354,ARO-14356,ARO-14355: introduce image registry capability behind feature gate #5456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions api/hypershift/v1beta1/hosted_controlplane.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ type HostedControlPlaneSpec struct {
// This field is optional and once set cannot be changed.
// +optional
// +openshift:enable:FeatureGate=DisableClusterCapabilities
Comment thread
flavianmissi marked this conversation as resolved.
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Capabilities is immutable. Changes might result in unpredictable and disruptive behavior."
Capabilities *Capabilities `json:"capabilities,omitempty"`
}

Expand Down
4 changes: 1 addition & 3 deletions api/hypershift/v1beta1/hostedcluster_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,7 +363,6 @@ const ImageRegistryCapability OptionalCapability = OptionalCapability(configv1.C

// capabilities allows disabling optional components at install time.
// Once set, it cannot be changed.
// +kubebuilder:validation:XValidation:rule="!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)", message="disabledCapabilities is required once set"
type Capabilities struct {
// disabledCapabilities when specified, sets the cluster version baselineCapabilitySet to None
// and sets all additionalEnabledCapabilities BUT the ones supplied in disabledCapabilities.
Expand All @@ -375,7 +374,6 @@ type Capabilities struct {
// Once set, this field cannot be changed.
//
// +listType=atomic
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="disabledCapabilities is immutable"
// +optional
DisabledCapabilities []OptionalCapability `json:"disabledCapabilities,omitempty"`
}
Expand All @@ -388,7 +386,6 @@ type Capabilities struct {
// +kubebuilder:validation:XValidation:rule=`self.platform.type == "Azure" ? self.services.exists(s, s.service == "Konnectivity" && s.servicePublishingStrategy.type == "Route" && s.servicePublishingStrategy.route.hostname != "") : true`,message="Azure platform requires Konnectivity Route service with a hostname to be defined"
// +kubebuilder:validation:XValidation:rule=`self.platform.type == "Azure" ? self.services.exists(s, s.service == "Ignition" && s.servicePublishingStrategy.type == "Route" && s.servicePublishingStrategy.route.hostname != "") : true`,message="Azure platform requires Ignition Route service with a hostname to be defined"
// +kubebuilder:validation:XValidation:rule=`has(self.issuerURL) || !has(self.serviceAccountSigningKey)`,message="If serviceAccountSigningKey is set, issuerURL must be set"

type HostedClusterSpec struct {
// release specifies the desired OCP release payload for all the hosted cluster components.
// This includes those components running management side like the Kube API Server and the CVO but also the operands which land in the hosted cluster data plane like the ingress controller, ovn agents, etc.
Expand Down Expand Up @@ -665,6 +662,7 @@ type HostedClusterSpec struct {
// This field is optional and once set cannot be changed.
// +optional
// +openshift:enable:FeatureGate=DisableClusterCapabilities
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Capabilities is immutable. Changes might result in unpredictable and disruptive behavior."
Capabilities *Capabilities `json:"capabilities,omitempty"`
}

Expand Down
8 changes: 3 additions & 5 deletions ...ated-crd-manifests/hostedclusters.hypershift.openshift.io/DisableClusterCapabilities.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,11 @@ spec:
type: string
type: array
x-kubernetes-list-type: atomic
x-kubernetes-validations:
- message: disabledCapabilities is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: disabledCapabilities is required once set
rule: '!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)'
- message: Capabilities is immutable. Changes might result in unpredictable
and disruptive behavior.
rule: self == oldSelf
channel:
description: |-
channel is an identifier for explicitly requesting that a non-default set of updates be applied to this cluster.
Expand Down
8 changes: 3 additions & 5 deletions ...crd-manifests/hostedcontrolplanes.hypershift.openshift.io/DisableClusterCapabilities.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,13 +138,11 @@ spec:
type: string
type: array
x-kubernetes-list-type: atomic
x-kubernetes-validations:
- message: disabledCapabilities is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: disabledCapabilities is required once set
rule: '!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)'
- message: Capabilities is immutable. Changes might result in unpredictable
and disruptive behavior.
rule: self == oldSelf
channel:
description: |-
channel is an identifier for explicitly requesting that a non-default
Expand Down
8 changes: 3 additions & 5 deletions ...ts/hypershift-operator/zz_generated.crd-manifests/hostedclusters-CustomNoUpgrade.crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,13 +218,11 @@ spec:
type: string
type: array
x-kubernetes-list-type: atomic
x-kubernetes-validations:
- message: disabledCapabilities is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: disabledCapabilities is required once set
rule: '!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)'
- message: Capabilities is immutable. Changes might result in unpredictable
and disruptive behavior.
rule: self == oldSelf
channel:
description: |-
channel is an identifier for explicitly requesting that a non-default set of updates be applied to this cluster.
Expand Down
8 changes: 3 additions & 5 deletions ...pershift-operator/zz_generated.crd-manifests/hostedclusters-TechPreviewNoUpgrade.crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -218,13 +218,11 @@ spec:
type: string
type: array
x-kubernetes-list-type: atomic
x-kubernetes-validations:
- message: disabledCapabilities is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: disabledCapabilities is required once set
rule: '!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)'
- message: Capabilities is immutable. Changes might result in unpredictable
and disruptive behavior.
rule: self == oldSelf
channel:
description: |-
channel is an identifier for explicitly requesting that a non-default set of updates be applied to this cluster.
Expand Down
8 changes: 3 additions & 5 deletions ...pershift-operator/zz_generated.crd-manifests/hostedcontrolplanes-CustomNoUpgrade.crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,13 +186,11 @@ spec:
type: string
type: array
x-kubernetes-list-type: atomic
x-kubernetes-validations:
- message: disabledCapabilities is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: disabledCapabilities is required once set
rule: '!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)'
- message: Capabilities is immutable. Changes might result in unpredictable
and disruptive behavior.
rule: self == oldSelf
channel:
description: |-
channel is an identifier for explicitly requesting that a non-default
Expand Down
8 changes: 3 additions & 5 deletions ...ift-operator/zz_generated.crd-manifests/hostedcontrolplanes-TechPreviewNoUpgrade.crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,13 +186,11 @@ spec:
type: string
type: array
x-kubernetes-list-type: atomic
x-kubernetes-validations:
- message: disabledCapabilities is immutable
rule: self == oldSelf
type: object
x-kubernetes-validations:
- message: disabledCapabilities is required once set
rule: '!has(oldSelf.disabledCapabilities) || has(self.disabledCapabilities)'
- message: Capabilities is immutable. Changes might result in unpredictable
and disruptive behavior.
rule: self == oldSelf
channel:
description: |-
channel is an identifier for explicitly requesting that a non-default
Expand Down
60 changes: 44 additions & 16 deletions control-plane-operator/controllers/hostedcontrolplane/cvo/reconcile.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package cvo

import (
"encoding/json"
"fmt"
"path"
"strings"
Expand All @@ -10,6 +11,7 @@ import (
"github.com/openshift/hypershift/control-plane-operator/controllers/hostedcontrolplane/manifests"
"github.com/openshift/hypershift/hypershift-operator/controllers/manifests/controlplaneoperator"
"github.com/openshift/hypershift/support/api"
"github.com/openshift/hypershift/support/capabilities"
"github.com/openshift/hypershift/support/certs"
"github.com/openshift/hypershift/support/config"
"github.com/openshift/hypershift/support/metrics"
Expand Down Expand Up @@ -118,7 +120,7 @@ func cvoLabels() map[string]string {

var port int32 = 8443

func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, controlPlaneReleaseImage, dataPlaneReleaseImage, cliImage, availabilityProberImage, clusterID string, updateService configv1.URL, platformType hyperv1.PlatformType, oauthEnabled, enableCVOManagementClusterMetricsAccess bool, featureSet configv1.FeatureSet) error {
func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, deploymentConfig config.DeploymentConfig, controlPlaneReleaseImage, dataPlaneReleaseImage, cliImage, availabilityProberImage, clusterID string, updateService configv1.URL, platformType hyperv1.PlatformType, oauthEnabled, enableCVOManagementClusterMetricsAccess bool, featureSet configv1.FeatureSet, caps *hyperv1.Capabilities) error {
ownerRef.ApplyTo(deployment)

// preserve existing resource requirements for main CVO container
Expand All @@ -132,6 +134,35 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
MatchLabels: cvoLabels(),
}
}

// the ClusterVersion resource is created by the CVO bootstrap container.
// we marshal it to json as a means to validate its formatting, which protects
// us against easily preventable mistakes, such as typos.
cv := &configv1.ClusterVersion{
TypeMeta: metav1.TypeMeta{
Kind: "ClusterVersion",
APIVersion: "config.openshift.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "version",
},
Spec: configv1.ClusterVersionSpec{
ClusterID: configv1.ClusterID(clusterID),
},
}

if !capabilities.IsImageRegistryCapabilityEnabled(caps) {
cv.Spec.Capabilities = &configv1.ClusterVersionCapabilitiesSpec{
BaselineCapabilitySet: configv1.ClusterVersionCapabilitySetNone,
AdditionalEnabledCapabilities: capabilities.CalculateEnabledCapabilities(caps),
}
}

clusterVersionJSON, err := json.Marshal(cv)
if err != nil {
return err
}

deployment.Spec = appsv1.DeploymentSpec{
Selector: selector,
Template: corev1.PodTemplateSpec{
Expand All @@ -142,7 +173,7 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
AutomountServiceAccountToken: ptr.To(false),
InitContainers: []corev1.Container{
util.BuildContainer(cvoContainerPrepPayload(), buildCVOContainerPrepPayload(dataPlaneReleaseImage, platformType, oauthEnabled, featureSet)),
util.BuildContainer(cvoContainerBootstrap(), buildCVOContainerBootstrap(cliImage, clusterID)),
util.BuildContainer(cvoContainerBootstrap(), buildCVOContainerBootstrap(cliImage, clusterVersionJSON)),
},
Containers: []corev1.Container{
util.BuildContainer(cvoContainerMain(), buildCVOContainerMain(controlPlaneReleaseImage, dataPlaneReleaseImage, deployment.Namespace, updateService, enableCVOManagementClusterMetricsAccess)),
Expand Down Expand Up @@ -203,13 +234,13 @@ func buildCVOContainerPrepPayload(image string, platformType hyperv1.PlatformTyp
}
}

func buildCVOContainerBootstrap(image, clusterID string) func(*corev1.Container) {
func buildCVOContainerBootstrap(image string, clusterVersionJSON []byte) func(*corev1.Container) {
return func(c *corev1.Container) {
c.Image = image
c.Command = []string{"/bin/bash"}
c.Args = []string{
"-c",
cvoBootrapScript(clusterID),
cvoBootstrapScript(clusterVersionJSON),
}
c.Resources.Requests = corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("10m"),
Expand Down Expand Up @@ -338,31 +369,28 @@ func preparePayloadScript(platformType hyperv1.PlatformType, oauthEnabled bool,
return strings.Join(stmts, "\n")
}

func cvoBootrapScript(clusterID string) string {
func cvoBootstrapScript(clusterVersionJSON []byte) string {
payloadDir := volumeMounts.Path(cvoContainerBootstrap().Name, cvoVolumePayload().Name)
var scriptTemplate = `#!/bin/bash
set -euo pipefail
cat > /tmp/clusterversion.yaml <<EOF
apiVersion: config.openshift.io/v1
kind: ClusterVersion
metadata:
name: version
spec:
clusterID: %s
MANIFEST_DIR=%s/manifests
cat > /tmp/clusterversion.json <<EOF
%s
EOF
oc get ns openshift-config &> /dev/null || oc create ns openshift-config
oc get ns openshift-config-managed &> /dev/null || oc create ns openshift-config-managed
oc apply -f ${MANIFEST_DIR}/0000_00_cluster-version-operator_01_clusterversions*
Comment thread
flavianmissi marked this conversation as resolved.
oc apply -f /tmp/clusterversion.json
while true; do
echo "Applying CVO bootstrap manifests"
if oc apply -f %s/manifests; then
echo "Applying CVO bootstrap manifests..."
if oc apply -f ${MANIFEST_DIR}; then
echo "Bootstrap manifests applied successfully."
break
fi
sleep 1
done
oc get clusterversion/version &> /dev/null || oc create -f /tmp/clusterversion.yaml
`
return fmt.Sprintf(scriptTemplate, clusterID, payloadDir)
return fmt.Sprintf(scriptTemplate, payloadDir, string(clusterVersionJSON))
}

func buildCVOContainerMain(controlPlaneReleaseImage, dataPlaneReleaseImage, namespace string, updateService configv1.URL, enableCVOManagementClusterMetricsAccess bool) func(c *corev1.Container) {
Expand Down
34 changes: 20 additions & 14 deletions control-plane-operator/controllers/hostedcontrolplane/hostedcontrolplane_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1331,11 +1331,12 @@ func (r *HostedControlPlaneReconciler) reconcile(ctx context.Context, hostedCont

if !r.IsCPOV2 {
// Reconcile image registry operator
r.Log.Info("Reconciling Image Registry Operator")
if err := r.reconcileImageRegistryOperator(ctx, hostedControlPlane, releaseImageProvider, userReleaseImageProvider, createOrUpdate); err != nil {
return fmt.Errorf("failed to reconcile image registry operator: %w", err)
if capabilities.IsImageRegistryCapabilityEnabled(hostedControlPlane.Spec.Capabilities) {
Comment thread
flavianmissi marked this conversation as resolved.
r.Log.Info("Reconciling Image Registry Operator")
if err := r.reconcileImageRegistryOperator(ctx, hostedControlPlane, releaseImageProvider, userReleaseImageProvider, createOrUpdate); err != nil {
return fmt.Errorf("failed to reconcile image registry operator: %w", err)
}
}

if IsStorageAndCSIManaged(hostedControlPlane) {
// Reconcile cluster storage operator
r.Log.Info("Reconciling cluster storage operator")
Expand Down Expand Up @@ -2544,12 +2545,14 @@ func (r *HostedControlPlaneReconciler) reconcilePKI(ctx context.Context, hcp *hy
return fmt.Errorf("failed to reconcile olm operator serving cert: %w", err)
}

// Image Registry Operator Serving Cert
imageRegistryOperatorServingCert := manifests.ImageRegistryOperatorServingCert(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, imageRegistryOperatorServingCert, func() error {
return pki.ReconcileRegistryOperatorServingCert(imageRegistryOperatorServingCert, rootCASecret, p.OwnerRef)
}); err != nil {
return fmt.Errorf("failed to reconcile image registry operator serving cert: %w", err)
if capabilities.IsImageRegistryCapabilityEnabled(hcp.Spec.Capabilities) {
// Image Registry Operator Serving Cert
imageRegistryOperatorServingCert := manifests.ImageRegistryOperatorServingCert(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, imageRegistryOperatorServingCert, func() error {
return pki.ReconcileRegistryOperatorServingCert(imageRegistryOperatorServingCert, rootCASecret, p.OwnerRef)
}); err != nil {
return fmt.Errorf("failed to reconcile image registry operator serving cert: %w", err)
}
}

kcmServerSecret := manifests.KCMServerCertSecret(hcp.Namespace)
Expand Down Expand Up @@ -3013,7 +3016,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex

kubeAPIServerConfig := manifests.KASConfig(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, kubeAPIServerConfig, func() error {
return kas.ReconcileConfig(kubeAPIServerConfig, p.OwnerRef, p.ConfigParams())
return kas.ReconcileConfig(kubeAPIServerConfig, p.OwnerRef, p.ConfigParams(), hcp.Spec.Capabilities)
}); err != nil {
return fmt.Errorf("failed to reconcile api server config: %w", err)
}
Expand Down Expand Up @@ -3349,7 +3352,7 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftAPIServer(ctx context.C
p := oapi.NewOpenShiftAPIServerParams(hcp, observedConfig, releaseImageProvider, r.SetDefaultSecurityContext)
oapicfg := manifests.OpenShiftAPIServerConfig(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, oapicfg, func() error {
return oapi.ReconcileConfig(oapicfg, p.AuditWebhookRef, p.OwnerRef, p.EtcdURL, p.IngressDomain(), p.MinTLSVersion(), p.CipherSuites(), p.Image, p.Project)
return oapi.ReconcileConfig(oapicfg, p.AuditWebhookRef, p.OwnerRef, p.EtcdURL, p.IngressDomain(), p.MinTLSVersion(), p.CipherSuites(), p.Image, p.Project, hcp.Spec.Capabilities)
}); err != nil {
return fmt.Errorf("failed to reconcile openshift apiserver config: %w", err)
}
Expand Down Expand Up @@ -3537,7 +3540,10 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftControllerManager(ctx c
p := ocm.NewOpenShiftControllerManagerParams(hcp, observedConfig, releaseImageProvider, r.SetDefaultSecurityContext)
config := manifests.OpenShiftControllerManagerConfig(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, config, func() error {
return ocm.ReconcileOpenShiftControllerManagerConfig(config, p.OwnerRef, p.DeployerImage, p.DockerBuilderImage, p.MinTLSVersion(), p.CipherSuites(), p.Image, p.Build, p.Network)
return ocm.ReconcileOpenShiftControllerManagerConfig(config,
p.OwnerRef, p.DeployerImage, p.DockerBuilderImage,
p.MinTLSVersion(), p.CipherSuites(), p.Image, p.Build,
p.Network, hcp.Spec.Capabilities)
}); err != nil {
return fmt.Errorf("failed to reconcile openshift controller manager config: %w", err)
}
Expand Down Expand Up @@ -3694,7 +3700,7 @@ func (r *HostedControlPlaneReconciler) reconcileClusterVersionOperator(ctx conte

deployment := manifests.ClusterVersionOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return cvo.ReconcileDeployment(deployment, p.OwnerRef, p.DeploymentConfig, controlPlaneReleaseImage, dataPlaneReleaseImage, p.CLIImage, p.AvailabilityProberImage, p.ClusterID, hcp.Spec.UpdateService, p.PlatformType, util.HCPOAuthEnabled(hcp), r.EnableCVOManagementClusterMetricsAccess, p.FeatureSet)
return cvo.ReconcileDeployment(deployment, p.OwnerRef, p.DeploymentConfig, controlPlaneReleaseImage, dataPlaneReleaseImage, p.CLIImage, p.AvailabilityProberImage, p.ClusterID, hcp.Spec.UpdateService, p.PlatformType, util.HCPOAuthEnabled(hcp), r.EnableCVOManagementClusterMetricsAccess, p.FeatureSet, hcp.Spec.Capabilities)
}); err != nil {
return fmt.Errorf("failed to reconcile cluster version operator deployment: %w", err)
}
Expand Down
Loading