Introduce node affinity awareness for apinet loadbalancers

apinetlet/controllers/controllers_suite_test.go

@@ -13,17 +13,15 @@ import (
 
 	"github.com/ironcore-dev/controller-utils/buildutils"
 	"github.com/ironcore-dev/controller-utils/modutils"
-	apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client"
-	apinetclient "github.com/ironcore-dev/ironcore-net/internal/client"
-
 	apinetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1"
+	apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client"
 	ironcorenet "github.com/ironcore-dev/ironcore-net/client-go/ironcorenet/versioned"
+	apinetclient "github.com/ironcore-dev/ironcore-net/internal/client"
 	ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1"
 	networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1"
 	envtestutils "github.com/ironcore-dev/ironcore/utils/envtest"
 	"github.com/ironcore-dev/ironcore/utils/envtest/apiserver"
-	. "github.com/onsi/ginkgo/v2"
-	. "github.com/onsi/gomega"
+
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
@@ -32,10 +30,13 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config"
 	"sigs.k8s.io/controller-runtime/pkg/envtest"
-	. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -187,10 +188,11 @@ func SetupTest(apiNetNamespace *corev1.Namespace) *corev1.Namespace {
 		}).SetupWithManager(k8sManager, k8sManager.GetCache())).To(Succeed())
 
 		Expect((&LoadBalancerReconciler{
-			Client:          k8sManager.GetClient(),
-			APINetClient:    k8sManager.GetClient(),
-			APINetInterface: apiNetInterface,
-			APINetNamespace: apiNetNamespace.Name,
+			Client:              k8sManager.GetClient(),
+			APINetClient:        k8sManager.GetClient(),
+			APINetInterface:     apiNetInterface,
+			APINetNamespace:     apiNetNamespace.Name,
+			IsNodeAffinityAware: true,
 		}).SetupWithManager(k8sManager, k8sManager.GetCache())).To(Succeed())
 
 		Expect((&NetworkPolicyReconciler{
@@ -206,7 +208,6 @@ func SetupTest(apiNetNamespace *corev1.Namespace) *corev1.Namespace {
 			defer GinkgoRecover()
 			Expect(k8sManager.Start(mgrCtx)).To(Succeed(), "failed to start manager")
 		}()
-
 	})
 
 	return apiNetNamespace

apinetlet/controllers/loadbalancer_controller.go

@@ -11,23 +11,23 @@ import (
 	"github.com/go-logr/logr"
 	"golang.org/x/exp/slices"
 
+	"github.com/ironcore-dev/controller-utils/clientutils"
 	apinetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1"
 	"github.com/ironcore-dev/ironcore-net/apimachinery/api/net"
 	apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client"
 	apinetlethandler "github.com/ironcore-dev/ironcore-net/apinetlet/handler"
 	"github.com/ironcore-dev/ironcore-net/apinetlet/provider"
 	apinetv1alpha1ac "github.com/ironcore-dev/ironcore-net/client-go/applyconfigurations/core/v1alpha1"
 	ironcorenet "github.com/ironcore-dev/ironcore-net/client-go/ironcorenet/versioned"
-	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
-
-	"github.com/ironcore-dev/controller-utils/clientutils"
 	commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1"
 	ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1"
 	networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1"
 	"github.com/ironcore-dev/ironcore/utils/predicates"
+
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	metav1ac "k8s.io/client-go/applyconfigurations/meta/v1"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
@@ -49,6 +49,8 @@ type LoadBalancerReconciler struct {
 	APINetNamespace string
 
 	WatchFilterValue string
+
+	IsNodeAffinityAware bool
 }
 
 //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch
@@ -151,14 +153,19 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger,
 		return ctrl.Result{}, nil
 	}
 
+	apiNetDestinations, err := r.prepareApiNetDestinations(ctx, loadBalancer)
+	if err != nil {
+		return ctrl.Result{}, fmt.Errorf("error preparing APINet destinations: %w", err)
+	}
+
 	log.V(1).Info("Applying APINet load balancer")
-	apiNetLoadBalancer, err := r.applyAPINetLoadBalancer(ctx, loadBalancer, apiNetNetworkName)
+	apiNetLoadBalancer, err := r.applyAPINetLoadBalancer(ctx, loadBalancer, apiNetDestinations, apiNetNetworkName)
 	if err != nil {
 		return ctrl.Result{}, fmt.Errorf("error applying APINet load balancer: %w", err)
 	}
 
 	log.V(1).Info("Manage APINet load balancer routing")
-	if err := r.manageAPINetLoadBalancerRouting(ctx, loadBalancer, apiNetLoadBalancer); err != nil {
+	if err := r.manageAPINetLoadBalancerRouting(ctx, loadBalancer, apiNetLoadBalancer, apiNetDestinations); err != nil {
 		return ctrl.Result{}, err
 	}
 
@@ -174,13 +181,13 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger,
 	return ctrl.Result{}, nil
 }
 
-func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetLoadBalancer *apinetv1alpha1.LoadBalancer) error {
+func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer) ([]apinetv1alpha1.LoadBalancerDestination, error) {
+	apiNetDsts := make([]apinetv1alpha1.LoadBalancerDestination, 0)
 	loadBalancerRouting := &networkingv1alpha1.LoadBalancerRouting{}
 	if err := r.Get(ctx, client.ObjectKeyFromObject(loadBalancer), loadBalancerRouting); client.IgnoreNotFound(err) != nil {
-		return fmt.Errorf("error getting load balancer routing: %w", err)
+		return nil, fmt.Errorf("error getting load balancer routing: %w", err)
 	}
 
-	apiNetDsts := make([]apinetv1alpha1.LoadBalancerDestination, 0)
 	for _, dst := range loadBalancerRouting.Destinations {
 		var apiNetTargetRef *apinetv1alpha1.LoadBalancerTargetRef
 		if targetRef := dst.TargetRef; targetRef != nil {
@@ -202,6 +209,10 @@ func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Con
 		})
 	}
 
+	return apiNetDsts, nil
+}
+
+func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetLoadBalancer *apinetv1alpha1.LoadBalancer, apiNetDsts []apinetv1alpha1.LoadBalancerDestination) error {
 	apiNetLoadBalancerRouting := &apinetv1alpha1.LoadBalancerRouting{
 		TypeMeta: metav1.TypeMeta{
 			APIVersion: apinetv1alpha1.SchemeGroupVersion.String(),
@@ -276,7 +287,7 @@ func (r *LoadBalancerReconciler) getInternalLoadBalancerAPINetIPs(ctx context.Co
 	return ips, nil
 }
 
-func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetNetworkName string) (*apinetv1alpha1.LoadBalancer, error) {
+func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetDestinations []apinetv1alpha1.LoadBalancerDestination, apiNetNetworkName string) (*apinetv1alpha1.LoadBalancer, error) {
 	apiNetLoadBalancerType, err := loadBalancerTypeToAPINetLoadBalancerType(loadBalancer.Spec.Type)
 	if err != nil {
 		return nil, err
@@ -315,6 +326,29 @@ func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, lo
 					),
 			),
 		)
+
+	if r.IsNodeAffinityAware {
+		if len(apiNetDestinations) > 0 {
+			nodeSelector := apinetv1alpha1ac.NodeSelector()
+			for i := range apiNetDestinations {
+				if apiNetDestinations[i].TargetRef != nil {
+					apiNetDestinationName := apiNetDestinations[i].TargetRef.NodeRef.Name
+					nodeSelector.WithNodeSelectorTerms(apinetv1alpha1ac.NodeSelectorTerm().WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement().
+						WithKey("metadata.name").
+						WithOperator(apinetv1alpha1.NodeSelectorOpIn).
+						WithValues([]string{apiNetDestinationName}...),
+					))
+				}
+			}
+			if len(nodeSelector.NodeSelectorTerms) > 0 {
+				apiNetLoadBalancerApplyCfg.Spec.Template.Spec.Affinity.
+					WithNodeAffinity(apinetv1alpha1ac.NodeAffinity().
+						WithRequiredDuringSchedulingIgnoredDuringExecution(nodeSelector),
+					)
+			}
+		}
+	}
+
 	apiNetLoadBalancer, err := r.APINetInterface.CoreV1alpha1().
 		LoadBalancers(r.APINetNamespace).
 		Apply(ctx, apiNetLoadBalancerApplyCfg, metav1.ApplyOptions{FieldManager: string(fieldOwner), Force: true})

apinetlet/controllers/loadbalancer_controller_test.go

@@ -4,19 +4,23 @@
 package controllers
 
 import (
+	"net/netip"
+
 	"github.com/ironcore-dev/ironcore-net/api/core/v1alpha1"
 	"github.com/ironcore-dev/ironcore-net/apimachinery/api/net"
 	apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client"
 	commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1"
 	ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1"
 	networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+
 	. "github.com/ironcore-dev/ironcore/utils/testing"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
-	corev1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	. "sigs.k8s.io/controller-runtime/pkg/envtest/komega"
 )
 
@@ -251,4 +255,118 @@ var _ = Describe("LoadBalancerController", func() {
 			}))),
 		)
 	})
+
+	It("should manage the APINet load balancer and its node affintity", func(ctx SpecContext) {
+		By("creating a load balancer")
+		loadBalancer := &networkingv1alpha1.LoadBalancer{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace:    ns.Name,
+				GenerateName: "load-balancer-",
+			},
+			Spec: networkingv1alpha1.LoadBalancerSpec{
+				Type:       networkingv1alpha1.LoadBalancerTypePublic,
+				IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
+				NetworkRef: corev1.LocalObjectReference{Name: network.Name},
+			},
+		}
+		Expect(k8sClient.Create(ctx, loadBalancer)).To(Succeed())
+		DeferCleanup(k8sClient.Delete, loadBalancer)
+
+		By("creating the load balancer routing")
+		lbRouting := &networkingv1alpha1.LoadBalancerRouting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      loadBalancer.Name,
+				Namespace: loadBalancer.Namespace,
+			},
+			NetworkRef: commonv1alpha1.LocalUIDReference{
+				Name: network.Name,
+				UID:  network.UID,
+			},
+			Destinations: []networkingv1alpha1.LoadBalancerDestination{
+				{
+					IP: commonv1alpha1.IP{Addr: netip.MustParseAddr("192.168.0.1")},
+					TargetRef: &networkingv1alpha1.LoadBalancerTargetRef{
+						UID:        "first-nic-uid",
+						Name:       "first-nic-name",
+						ProviderID: "ironcore-net://namespace/first-apinet-nic-name/first-node-name/first-metalnet-nic-uid",
+					},
+				},
+				{
+					IP: commonv1alpha1.IP{Addr: netip.MustParseAddr("192.168.0.2")},
+					TargetRef: &networkingv1alpha1.LoadBalancerTargetRef{
+						UID:        "second-nic-uid",
+						Name:       "second-nic-name",
+						ProviderID: "ironcore-net://namespace/second-apinet-nic-name/second-node-name/second-metalnet-nic-uid",
+					},
+				},
+			},
+		}
+		Expect(k8sClient.Create(ctx, lbRouting)).To(Succeed())
+		DeferCleanup(k8sClient.Delete, lbRouting)
+
+		By("waiting for the APINet load balancer to exist")
+		apiNetLoadBalancer := &v1alpha1.LoadBalancer{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: apiNetNs.Name,
+				Name:      string(loadBalancer.UID),
+			},
+		}
+		Eventually(Object(apiNetLoadBalancer)).Should(SatisfyAll(
+			HaveField("Labels", apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer)),
+			HaveField("Spec", MatchFields(IgnoreExtras, Fields{
+				"Type":       Equal(v1alpha1.LoadBalancerTypePublic),
+				"NetworkRef": Equal(corev1.LocalObjectReference{Name: apiNetNetwork.Name}),
+				"IPs": ConsistOf(MatchFields(IgnoreExtras, Fields{
+					"IPFamily": Equal(corev1.IPv4Protocol),
+					"Name":     Equal("ipv4"),
+				})),
+				"Selector": Equal(&metav1.LabelSelector{
+					MatchLabels: apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer),
+				}),
+				"Template": Equal(v1alpha1.InstanceTemplate{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer),
+					},
+					Spec: v1alpha1.InstanceSpec{
+						Affinity: &v1alpha1.Affinity{
+							InstanceAntiAffinity: &v1alpha1.InstanceAntiAffinity{
+								RequiredDuringSchedulingIgnoredDuringExecution: []v1alpha1.InstanceAffinityTerm{
+									{
+										LabelSelector: &metav1.LabelSelector{
+											MatchLabels: apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer),
+										},
+										TopologyKey: v1alpha1.TopologyZoneLabel,
+									},
+								},
+							},
+							NodeAffinity: &v1alpha1.NodeAffinity{
+								RequiredDuringSchedulingIgnoredDuringExecution: &v1alpha1.NodeSelector{
+									NodeSelectorTerms: []v1alpha1.NodeSelectorTerm{
+										{
+											MatchFields: []v1alpha1.NodeSelectorRequirement{
+												{
+													Key:      "metadata.name",
+													Operator: v1alpha1.NodeSelectorOpIn,
+													Values:   []string{"first-node-name"},
+												},
+											},
+										},
+										{
+											MatchFields: []v1alpha1.NodeSelectorRequirement{
+												{
+													Key:      "metadata.name",
+													Operator: v1alpha1.NodeSelectorOpIn,
+													Values:   []string{"second-node-name"},
+												},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				}),
+			}))),
+		)
+	})
 })