Set load balancer target nodes in request serving isolation mode

Adds an annotation on the HostedCluster to allow setting target node
labels for control plane router load balancers.
Modifies the request serving node scheduler to set this annotation when
scheduling a HostedCluster on a set of nodes.
Only applies subnet annotation to private router loadbalancer, since
public load balancers require public subnets.

api/hypershift/v1beta1/hostedcluster_types.go

@@ -264,9 +264,13 @@ const (
 	KubeAPIServerGOMemoryLimitAnnotation = "hypershift.openshift.io/kube-apiserver-gomemlimit"
 
 	// AWSLoadBalancerSubnetsAnnotation allows specifying the subnets to use for control plane load balancers
-	// in the AWS platform.
+	// in the AWS platform. These subnets only apply to private load balancers.
 	AWSLoadBalancerSubnetsAnnotation = "hypershift.openshift.io/aws-load-balancer-subnets"
 
+	// AWSLoadBalancerTargetNodesAnnotation allows specifying label selectors to choose target nodes for
+	// control plane load balancers in the AWS platform.
+	AWSLoadBalancerTargetNodesAnnotation = "hypershift.openshift.io/aws-load-balancer-target-node-labels"
+
 	// DisableClusterAutoscalerAnnotation allows disabling the cluster autoscaler for a hosted cluster.
 	// This annotation is only set by the hypershift-operator on HosterControlPlanes.
 	// It is not set by the end-user.

control-plane-operator/controllers/hostedcontrolplane/ingress/router.go

@@ -236,7 +236,12 @@ func ReconcileRouterService(svc *corev1.Service, internal, crossZoneLoadBalancin
 		if crossZoneLoadBalancingEnabled {
 			svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-cross-zone-load-balancing-enabled"] = "true"
 		}
-		util.ApplyAWSLoadBalancerSubnetsAnnotation(svc, hcp)
+		if internal {
+			// Only the internal nlb should get private subnets assigned.
+			// The external nlb should use the default public subnets.
+			util.ApplyAWSLoadBalancerSubnetsAnnotation(svc, hcp)
+		}
+		util.ApplyAWSLoadBalancerTargetNodesAnnotation(svc, hcp)
 	}
 
 	if svc.Labels == nil {

control-plane-operator/controllers/hostedcontrolplane/kas/service.go

@@ -54,7 +54,6 @@ func ReconcileService(svc *corev1.Service, strategy *hyperv1.ServicePublishingSt
 	}
 	if hcp.Spec.Platform.Type == hyperv1.AWSPlatform {
 		svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-type"] = "nlb"
-		util.ApplyAWSLoadBalancerSubnetsAnnotation(svc, hcp)
 	}
 	switch strategy.Type {
 	case hyperv1.LoadBalancer:
@@ -272,7 +271,6 @@ func ReconcileKonnectivityServerService(svc *corev1.Service, ownerRef config.Own
 			}
 			svc.Annotations[hyperv1.ExternalDNSHostnameAnnotation] = strategy.LoadBalancer.Hostname
 		}
-		util.ApplyAWSLoadBalancerSubnetsAnnotation(svc, hcp)
 	case hyperv1.NodePort:
 		svc.Spec.Type = corev1.ServiceTypeNodePort
 		if portSpec.NodePort == 0 && strategy.NodePort != nil {

hypershift-operator/controllers/hostedcluster/hostedcluster_controller.go

@@ -1793,6 +1793,7 @@ func reconcileHostedControlPlane(hcp *hyperv1.HostedControlPlane, hcluster *hype
 		hyperv1.KubeAPIServerGOMemoryLimitAnnotation,
 		hyperv1.RequestServingNodeAdditionalSelectorAnnotation,
 		hyperv1.AWSLoadBalancerSubnetsAnnotation,
+		hyperv1.AWSLoadBalancerTargetNodesAnnotation,
 	}
 	for _, key := range mirroredAnnotations {
 		val, hasVal := hcluster.Annotations[key]

hypershift-operator/controllers/scheduler/dedicated_request_serving_nodes.go

@@ -285,6 +285,7 @@ func (r *DedicatedServingComponentScheduler) Reconcile(ctx context.Context, req
 	if lbSubnets != "" {
 		hcluster.Annotations[hyperv1.AWSLoadBalancerSubnetsAnnotation] = lbSubnets
 	}
+	hcluster.Annotations[hyperv1.AWSLoadBalancerTargetNodesAnnotation] = fmt.Sprintf("%s=%s,%s=%s", hyperv1.HostedClusterLabel, clusterKey(hcluster), hyperv1.RequestServingComponentLabel, "true")
 	if err := r.Patch(ctx, hcluster, client.MergeFrom(originalHcluster)); err != nil {
 		return ctrl.Result{}, fmt.Errorf("failed to update hostedcluster annotation: %w", err)
 	}
@@ -630,6 +631,8 @@ func (r *DedicatedServingComponentSchedulerAndSizer) updateHostedCluster(ctx con
 		hc.Annotations[hyperv1.AWSLoadBalancerSubnetsAnnotation] = lbSubnets
 	}
 
+	hc.Annotations[hyperv1.AWSLoadBalancerTargetNodesAnnotation] = fmt.Sprintf("%s=%s,%s=%s", hyperv1.HostedClusterLabel, clusterKey(hc), hyperv1.RequestServingComponentLabel, "true")
+
 	hc.Annotations[hyperv1.RequestServingNodeAdditionalSelectorAnnotation] = fmt.Sprintf("%s=%s", hyperv1.NodeSizeLabel, size)
 
 	if !equality.Semantic.DeepEqual(hc, original) {

support/util/util.go

@@ -344,6 +344,19 @@ func ApplyAWSLoadBalancerSubnetsAnnotation(svc *corev1.Service, hcp *hyperv1.Hos
 	}
 }
 
+func ApplyAWSLoadBalancerTargetNodesAnnotation(svc *corev1.Service, hcp *hyperv1.HostedControlPlane) {
+	if hcp.Spec.Platform.Type != hyperv1.AWSPlatform {
+		return
+	}
+	if svc.Annotations == nil {
+		svc.Annotations = make(map[string]string)
+	}
+	selectors, ok := hcp.Annotations[hyperv1.AWSLoadBalancerTargetNodesAnnotation]
+	if ok {
+		svc.Annotations["service.beta.kubernetes.io/aws-load-balancer-target-node-labels"] = selectors
+	}
+}
+
 func DoesMgmtClusterAndNodePoolCPUArchMatch(mgmtClusterCPUArch, nodePoolArch string) error {
 	if mgmtClusterCPUArch != nodePoolArch {
 		return fmt.Errorf("multi-arch hosted cluster is not enabled and management cluster and nodepool cpu arches do not match - management cluster cpu arch: %s, nodepool cpu arch: %s", mgmtClusterCPUArch, nodePoolArch)