Skip to content

Commit

Permalink
use 2040 for apiserver svc in IBM provider
Browse files Browse the repository at this point in the history
  • Loading branch information
isco-rodriguez committed Feb 16, 2024
1 parent 15533d4 commit d6bcdf6
Show file tree
Hide file tree
Showing 26 changed files with 99 additions and 63 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ func ReconcileAutoscalerDeployment(deployment *appsv1.Deployment, hcp *hyperv1.H
},
}

util.AvailabilityProber(kas.InClusterKASReadyURL(), availabilityProberImage, &deployment.Spec.Template.Spec)
util.AvailabilityProber(kas.InClusterKASReadyURL(hcp.Spec.Platform.Type), availabilityProberImage, &deployment.Spec.Template.Spec)

deploymentConfig := config.DeploymentConfig{
AdditionalLabels: map[string]string{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ var (
}
)

func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, image string, deploymentConfig config.DeploymentConfig, availabilityProberImage string) error {
func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef, image string, deploymentConfig config.DeploymentConfig, availabilityProberImage string, platformType hyperv1.PlatformType) error {
// preserve existing resource requirements for main CPC container
mainContainer := util.FindContainer(cpcContainerMain().Name, deployment.Spec.Template.Spec.Containers)
if mainContainer != nil {
Expand Down Expand Up @@ -64,7 +64,7 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
deployment.Spec.Template.Spec.AutomountServiceAccountToken = pointer.Bool(false)
deploymentConfig.ApplyTo(deployment)

util.AvailabilityProber(kas.InClusterKASReadyURL(), availabilityProberImage, &deployment.Spec.Template.Spec)
util.AvailabilityProber(kas.InClusterKASReadyURL(platformType), availabilityProberImage, &deployment.Spec.Template.Spec)
return nil
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ func ReconcileServiceAccount(sa *corev1.ServiceAccount, ownerRef config.OwnerRef
return nil
}

func ReconcileDeployment(dep *appsv1.Deployment, params Params) error {
func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) error {
params.OwnerRef.ApplyTo(dep)

dep.Spec.Replicas = utilpointer.Int32(1)
Expand Down Expand Up @@ -559,7 +559,7 @@ if [[ -n $sc ]]; then kubectl --kubeconfig $kc delete --ignore-not-found validat
}

params.DeploymentConfig.ApplyTo(dep)
util.AvailabilityProber(kas.InClusterKASReadyURL(), params.AvailabilityProberImage, &dep.Spec.Template.Spec, func(o *util.AvailabilityProberOpts) {
util.AvailabilityProber(kas.InClusterKASReadyURL(platformType), params.AvailabilityProberImage, &dep.Spec.Template.Spec, func(o *util.AvailabilityProberOpts) {
o.KubeconfigVolumeName = "hosted-etc-kube"
o.RequiredAPIs = []schema.GroupVersionKind{
{Group: "operator.openshift.io", Version: "v1", Kind: "Network"},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ func TestReconcileDeployment(t *testing.T) {
}

dep := &appsv1.Deployment{}
if err := ReconcileDeployment(dep, tc.params); err != nil {
if err := ReconcileDeployment(dep, tc.params, hyperv1.NonePlatform); err != nil {
t.Fatalf("ReconcileDeployment: %v", err)
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@ func ReconcileDeployment(deployment *appsv1.Deployment, image, hcpName, openShif
}

deploymentConfig.ApplyTo(deployment)
util.AvailabilityProber(kas.InClusterKASReadyURL(), availabilityProberImage, &deployment.Spec.Template.Spec, func(o *util.AvailabilityProberOpts) {
util.AvailabilityProber(kas.InClusterKASReadyURL(hcp.Spec.Platform.Type), availabilityProberImage, &deployment.Spec.Template.Spec, func(o *util.AvailabilityProberOpts) {
o.KubeconfigVolumeName = "kubeconfig"
o.RequiredAPIs = []schema.GroupVersionKind{
{Group: "imageregistry.operator.openshift.io", Version: "v1", Kind: "Config"},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func ReconcileDeployment(deployment *appsv1.Deployment, ownerRef config.OwnerRef
}
deploymentConfig.ApplyTo(deployment)
util.AvailabilityProber(
kas.InClusterKASReadyURL(),
kas.InClusterKASReadyURL(platformType),
availabilityProberImage,
&deployment.Spec.Template.Spec,
func(o *util.AvailabilityProberOpts) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv
// hosted-cluster services, or external services, so the operator does not
// require any special proxy configuration or permissions in the management
// cluster.
func ReconcileDeployment(dep *appsv1.Deployment, params Params) {
func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) {
dep.Spec.Selector = &metav1.LabelSelector{
MatchLabels: map[string]string{"name": "dns-operator"},
}
Expand Down Expand Up @@ -161,7 +161,7 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params) {
},
}}
util.AvailabilityProber(
kas.InClusterKASReadyURL(),
kas.InClusterKASReadyURL(platformType),
params.AvailabilityProberImage,
&dep.Spec.Template.Spec,
func(o *util.AvailabilityProberOpts) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -767,6 +767,9 @@ func (r *HostedControlPlaneReconciler) healthCheckKASLoadBalancers(ctx context.C
case !util.IsPublicHCP(hcp):
// When the cluster is private, checking the load balancers will depend on whether the load balancer is
// using the right subnets. To avoid uncertainty, we'll limit the check to the service endpoint.
if hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
return healthCheckKASEndpoint(manifests.KubeAPIServerService("").Name, config.KASSVCIBMCloudPort)
}
return healthCheckKASEndpoint(manifests.KubeAPIServerService("").Name, config.KASSVCPort)
case serviceStrategy.Type == hyperv1.Route:
externalRoute := manifests.KubeAPIServerExternalPublicRoute(hcp.Namespace)
Expand All @@ -781,6 +784,9 @@ func (r *HostedControlPlaneReconciler) healthCheckKASLoadBalancers(ctx context.C
case serviceStrategy.Type == hyperv1.LoadBalancer:
svc := manifests.KubeAPIServerService(hcp.Namespace)
port := config.KASSVCPort
if hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
port = config.KASSVCIBMCloudPort
}
if hcp.Spec.Platform.Type == hyperv1.AzurePlatform {
// If Azure we get the SVC handling the LB.
// TODO(alberto): remove this hack when having proper traffic management for Azure.
Expand Down Expand Up @@ -1211,6 +1217,9 @@ func (r *HostedControlPlaneReconciler) reconcileAPIServerService(ctx context.Con
p := kas.NewKubeAPIServerServiceParams(hcp)
apiServerService := manifests.KubeAPIServerService(hcp.Namespace)
kasSVCPort := config.KASSVCPort
if hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
kasSVCPort = config.KASSVCIBMCloudPort
}
if hcp.Spec.Platform.Type == hyperv1.AzurePlatform {
// For Azure we currently hardcode 7443 for the SVC LB as 6443 collides with public LB rule for the management cluster.
// https://bugzilla.redhat.com/show_bug.cgi?id=2060650
Expand Down Expand Up @@ -1634,6 +1643,9 @@ func (r *HostedControlPlaneReconciler) reconcileAPIServerServiceStatus(ctx conte
}

kasSVCLBPort := config.KASSVCPort
if hcp.Spec.Platform.Type == hyperv1.IBMCloudPlatform {
kasSVCLBPort = config.KASSVCIBMCloudPort
}
if hcp.Spec.Platform.Type == hyperv1.AzurePlatform {
// If Azure we get the SVC handling the LB.
// TODO(alberto): remove this hack when having proper traffic management for Azure.
Expand Down Expand Up @@ -2314,7 +2326,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex

serviceKubeconfigSecret := manifests.KASServiceKubeconfigSecret(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, serviceKubeconfigSecret, func() error {
return kas.ReconcileServiceKubeconfigSecret(serviceKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef)
return kas.ReconcileServiceKubeconfigSecret(serviceKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile service admin kubeconfig secret: %w", err)
}
Expand All @@ -2325,7 +2337,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeAPIServer(ctx context.Contex
if _, err := createOrUpdate(ctx, r, capiKubeconfigSecret, func() error {
// TODO(alberto): This secret is currently using the cluster-admin kubeconfig for the guest cluster.
// We should create a separate kubeconfig with a tight set of permissions for it to use.
return kas.ReconcileServiceCAPIKubeconfigSecret(capiKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef)
return kas.ReconcileServiceCAPIKubeconfigSecret(capiKubeconfigSecret, clientCertSecret, rootCA, p.OwnerRef, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile CAPI service admin kubeconfig secret: %w", err)
}
Expand Down Expand Up @@ -2576,6 +2588,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeControllerManager(ctx contex
clientCertSecret,
rootCAConfigMap,
p.OwnerRef,
hcp.Spec.Platform.Type,
)
}); err != nil {
return fmt.Errorf("failed to reconcile secret '%s/%s': %v", kcmKubeconfigSecret.Namespace, kcmKubeconfigSecret.Name, err)
Expand All @@ -2598,7 +2611,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeControllerManager(ctx contex
}

if _, err := createOrUpdate(ctx, r, kcmDeployment, func() error {
return kcm.ReconcileDeployment(kcmDeployment, kcmConfig, rootCAConfigMap, serviceServingCA, p)
return kcm.ReconcileDeployment(kcmDeployment, kcmConfig, rootCAConfigMap, serviceServingCA, p, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile kcm deployment: %w", err)
}
Expand Down Expand Up @@ -2626,6 +2639,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeScheduler(ctx context.Contex
clientCertSecret,
rootCA,
p.OwnerRef,
hcp.Spec.Platform.Type,
)
}); err != nil {
return fmt.Errorf("failed to reconcile secret '%s/%s': %v", schedulerKubeconfigSecret.Namespace, schedulerKubeconfigSecret.Name, err)
Expand All @@ -2639,7 +2653,7 @@ func (r *HostedControlPlaneReconciler) reconcileKubeScheduler(ctx context.Contex
}

if _, err := createOrUpdate(ctx, r, schedulerDeployment, func() error {
return scheduler.ReconcileDeployment(schedulerDeployment, p.OwnerRef, p.DeploymentConfig, p.HyperkubeImage, p.FeatureGates(), p.SchedulerPolicy(), p.AvailabilityProberImage, p.CipherSuites(), p.MinTLSVersion(), p.DisableProfiling, schedulerConfig)
return scheduler.ReconcileDeployment(schedulerDeployment, p.OwnerRef, p.DeploymentConfig, p.HyperkubeImage, p.FeatureGates(), p.SchedulerPolicy(), p.AvailabilityProberImage, p.CipherSuites(), p.MinTLSVersion(), p.DisableProfiling, schedulerConfig, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile scheduler deployment: %w", err)
}
Expand Down Expand Up @@ -2690,7 +2704,7 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftAPIServer(ctx context.C
}

if _, err := createOrUpdate(ctx, r, deployment, func() error {
return oapi.ReconcileDeployment(deployment, p.AuditWebhookRef, p.OwnerRef, oapicfg, serviceServingCA, p.OpenShiftAPIServerDeploymentConfig, p.OpenShiftAPIServerImage, p.ProxyImage, p.EtcdURL, p.AvailabilityProberImage)
return oapi.ReconcileDeployment(deployment, p.AuditWebhookRef, p.OwnerRef, oapicfg, serviceServingCA, p.OpenShiftAPIServerDeploymentConfig, p.OpenShiftAPIServerImage, p.ProxyImage, p.EtcdURL, p.AvailabilityProberImage, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile openshift apiserver deployment: %w", err)
}
Expand Down Expand Up @@ -2718,7 +2732,7 @@ func (r *HostedControlPlaneReconciler) reconcileOpenShiftOAuthAPIServer(ctx cont

deployment := manifests.OpenShiftOAuthAPIServerDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return oapi.ReconcileOAuthAPIServerDeployment(deployment, p.OwnerRef, p.OAuthAPIServerDeploymentParams(hcp))
return oapi.ReconcileOAuthAPIServerDeployment(deployment, p.OwnerRef, p.OAuthAPIServerDeploymentParams(hcp), hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile openshift oauth apiserver deployment: %w", err)
}
Expand Down Expand Up @@ -2779,7 +2793,7 @@ func (r *HostedControlPlaneReconciler) reconcileOAuthServer(ctx context.Context,

deployment := manifests.OAuthServerDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return oauth.ReconcileDeployment(ctx, r, deployment, p.OwnerRef, oauthConfig, p.OAuthServerImage, p.DeploymentConfig, p.IdentityProviders(), p.OauthConfigOverrides, p.AvailabilityProberImage, p.NamedCertificates(), p.Socks5ProxyImage, p.NoProxy)
return oauth.ReconcileDeployment(ctx, r, deployment, p.OwnerRef, oauthConfig, p.OAuthServerImage, p.DeploymentConfig, p.IdentityProviders(), p.OauthConfigOverrides, p.AvailabilityProberImage, p.NamedCertificates(), p.Socks5ProxyImage, p.NoProxy, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile oauth deployment: %w", err)
}
Expand Down Expand Up @@ -2866,7 +2880,7 @@ func (r *HostedControlPlaneReconciler) reconcileClusterPolicyController(ctx cont

deployment := manifests.ClusterPolicyControllerDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return clusterpolicy.ReconcileDeployment(deployment, p.OwnerRef, p.Image, p.DeploymentConfig, p.AvailabilityProberImage)
return clusterpolicy.ReconcileDeployment(deployment, p.OwnerRef, p.Image, p.DeploymentConfig, p.AvailabilityProberImage, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile cluster policy controller deployment: %w", err)
}
Expand Down Expand Up @@ -2925,7 +2939,7 @@ func (r *HostedControlPlaneReconciler) reconcileClusterNetworkOperator(ctx conte

deployment := manifests.ClusterNetworkOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return cno.ReconcileDeployment(deployment, p)
return cno.ReconcileDeployment(deployment, p, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile cluster network operator deployment: %w", err)
}
Expand Down Expand Up @@ -3016,7 +3030,7 @@ func (r *HostedControlPlaneReconciler) reconcileDNSOperator(ctx context.Context,

deployment := manifests.DNSOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
dnsoperator.ReconcileDeployment(deployment, p)
dnsoperator.ReconcileDeployment(deployment, p, hcp.Spec.Platform.Type)
return nil
}); err != nil {
return fmt.Errorf("failed to reconcile dnsoperator deployment: %w", err)
Expand Down Expand Up @@ -3046,7 +3060,7 @@ func (r *HostedControlPlaneReconciler) reconcileIngressOperator(ctx context.Cont

deployment := manifests.IngressOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
ingressoperator.ReconcileDeployment(deployment, p)
ingressoperator.ReconcileDeployment(deployment, p, hcp.Spec.Platform.Type)
return nil
}); err != nil {
return fmt.Errorf("failed to reconcile ingressoperator deployment: %w", err)
Expand Down Expand Up @@ -3151,7 +3165,7 @@ func (r *HostedControlPlaneReconciler) reconcileOperatorLifecycleManager(ctx con
}
catalogOperatorDeployment := manifests.CatalogOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, catalogOperatorDeployment, func() error {
return olm.ReconcileCatalogOperatorDeployment(catalogOperatorDeployment, p.OwnerRef, p.OLMImage, p.ProxyImage, p.OperatorRegistryImage, p.ReleaseVersion, p.DeploymentConfig, p.AvailabilityProberImage, p.NoProxy)
return olm.ReconcileCatalogOperatorDeployment(catalogOperatorDeployment, p.OwnerRef, p.OLMImage, p.ProxyImage, p.OperatorRegistryImage, p.ReleaseVersion, p.DeploymentConfig, p.AvailabilityProberImage, p.NoProxy, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile catalog operator deployment: %w", err)
}
Expand All @@ -3172,14 +3186,14 @@ func (r *HostedControlPlaneReconciler) reconcileOperatorLifecycleManager(ctx con

olmOperatorDeployment := manifests.OLMOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, olmOperatorDeployment, func() error {
return olm.ReconcileOLMOperatorDeployment(olmOperatorDeployment, p.OwnerRef, p.OLMImage, p.ProxyImage, p.ReleaseVersion, p.DeploymentConfig, p.AvailabilityProberImage, p.NoProxy)
return olm.ReconcileOLMOperatorDeployment(olmOperatorDeployment, p.OwnerRef, p.OLMImage, p.ProxyImage, p.ReleaseVersion, p.DeploymentConfig, p.AvailabilityProberImage, p.NoProxy, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile olm operator deployment: %w", err)
}

packageServerDeployment := manifests.OLMPackageServerDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, packageServerDeployment, func() error {
return olm.ReconcilePackageServerDeployment(packageServerDeployment, p.OwnerRef, p.OLMImage, p.ProxyImage, p.ReleaseVersion, p.PackageServerConfig, p.AvailabilityProberImage, p.NoProxy)
return olm.ReconcilePackageServerDeployment(packageServerDeployment, p.OwnerRef, p.OLMImage, p.ProxyImage, p.ReleaseVersion, p.PackageServerConfig, p.AvailabilityProberImage, p.NoProxy, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile packageserver deployment: %w", err)
}
Expand Down Expand Up @@ -3931,7 +3945,7 @@ func (r *HostedControlPlaneReconciler) reconcileCSISnapshotControllerOperator(ct

deployment := manifests.CSISnapshotControllerOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return snapshotcontroller.ReconcileOperatorDeployment(deployment, params)
return snapshotcontroller.ReconcileOperatorDeployment(deployment, params, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile CSI snapshot controller operator deployment: %w", err)
}
Expand Down Expand Up @@ -3967,7 +3981,7 @@ func (r *HostedControlPlaneReconciler) reconcileClusterStorageOperator(ctx conte

deployment := manifests.ClusterStorageOperatorDeployment(hcp.Namespace)
if _, err := createOrUpdate(ctx, r, deployment, func() error {
return storage.ReconcileOperatorDeployment(deployment, params)
return storage.ReconcileOperatorDeployment(deployment, params, hcp.Spec.Platform.Type)
}); err != nil {
return fmt.Errorf("failed to reconcile cluster storage operator deployment: %w", err)
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func NewParams(hcp *hyperv1.HostedControlPlane, version string, releaseImageProv
return p
}

func ReconcileDeployment(dep *appsv1.Deployment, params Params) {
func ReconcileDeployment(dep *appsv1.Deployment, params Params, platformType hyperv1.PlatformType) {
dep.Spec.Replicas = utilpointer.Int32(1)
dep.Spec.Selector = &metav1.LabelSelector{MatchLabels: map[string]string{"name": operatorName}}
dep.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType
Expand Down Expand Up @@ -203,7 +203,7 @@ func ReconcileDeployment(dep *appsv1.Deployment, params Params) {
}

util.AvailabilityProber(
kas.InClusterKASReadyURL(),
kas.InClusterKASReadyURL(platformType),
params.AvailabilityProberImage,
&dep.Spec.Template.Spec,
func(o *util.AvailabilityProberOpts) {
Expand Down

0 comments on commit d6bcdf6

Please sign in to comment.