From 07d19813de2b6a1651cdf004403783edf5bd9bad Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 10:20:58 +0200 Subject: [PATCH 1/9] introduce flag for node affinity awareness & extract destination preparation Signed-off-by: Benjamin Alpert --- .../controllers/loadbalancer_controller.go | 30 ++++++++++++++----- cmd/apinetlet/main.go | 24 ++++++++------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller.go b/apinetlet/controllers/loadbalancer_controller.go index a98f6291..2c5ec0eb 100644 --- a/apinetlet/controllers/loadbalancer_controller.go +++ b/apinetlet/controllers/loadbalancer_controller.go @@ -11,6 +11,7 @@ import ( "github.com/go-logr/logr" "golang.org/x/exp/slices" + "github.com/ironcore-dev/controller-utils/clientutils" apinetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" "github.com/ironcore-dev/ironcore-net/apimachinery/api/net" apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" @@ -18,16 +19,15 @@ import ( "github.com/ironcore-dev/ironcore-net/apinetlet/provider" apinetv1alpha1ac "github.com/ironcore-dev/ironcore-net/client-go/applyconfigurations/core/v1alpha1" ironcorenet "github.com/ironcore-dev/ironcore-net/client-go/ironcorenet/versioned" - metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" - - "github.com/ironcore-dev/controller-utils/clientutils" commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" "github.com/ironcore-dev/ironcore/utils/predicates" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1ac "k8s.io/client-go/applyconfigurations/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -49,6 +49,8 @@ type LoadBalancerReconciler struct { APINetNamespace string WatchFilterValue string + + IsNodeAffinityAware bool } //+kubebuilder:rbac:groups="",resources=events,verbs=create;patch @@ -151,6 +153,11 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } + _, apiNetDestinations, err := r.prepareApiNetDestinations(ctx, loadBalancer) + if err != nil { + return ctrl.Result{}, fmt.Errorf("error preparing APINet destinations: %w", err) + } + log.V(1).Info("Applying APINet load balancer") apiNetLoadBalancer, err := r.applyAPINetLoadBalancer(ctx, loadBalancer, apiNetNetworkName) if err != nil { @@ -158,7 +165,7 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, } log.V(1).Info("Manage APINet load balancer routing") - if err := r.manageAPINetLoadBalancerRouting(ctx, loadBalancer, apiNetLoadBalancer); err != nil { + if err := r.manageAPINetLoadBalancerRouting(ctx, loadBalancer, apiNetLoadBalancer, apiNetDestinations); err != nil { return ctrl.Result{}, err } @@ -174,13 +181,13 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } -func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetLoadBalancer *apinetv1alpha1.LoadBalancer) error { +func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer) (*networkingv1alpha1.LoadBalancerRouting, []apinetv1alpha1.LoadBalancerDestination, error) { + apiNetDsts := make([]apinetv1alpha1.LoadBalancerDestination, 0) loadBalancerRouting := &networkingv1alpha1.LoadBalancerRouting{} if err := r.Get(ctx, client.ObjectKeyFromObject(loadBalancer), loadBalancerRouting); client.IgnoreNotFound(err) != nil { - return fmt.Errorf("error getting load balancer routing: %w", err) + return nil, nil, fmt.Errorf("error getting load balancer routing: %w", err) } - apiNetDsts := make([]apinetv1alpha1.LoadBalancerDestination, 0) for _, dst := range loadBalancerRouting.Destinations { var apiNetTargetRef *apinetv1alpha1.LoadBalancerTargetRef if targetRef := dst.TargetRef; targetRef != nil { @@ -202,6 +209,10 @@ func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Con }) } + return loadBalancerRouting, apiNetDsts, nil +} + +func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetLoadBalancer *apinetv1alpha1.LoadBalancer, apiNetDsts []apinetv1alpha1.LoadBalancerDestination) error { apiNetLoadBalancerRouting := &apinetv1alpha1.LoadBalancerRouting{ TypeMeta: metav1.TypeMeta{ APIVersion: apinetv1alpha1.SchemeGroupVersion.String(), @@ -315,6 +326,11 @@ func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, lo ), ), ) + + if r.IsNodeAffinityAware { + // // TODO(balpert): extend the apiNetLoadBalancerApplyCfg with nodeaffinity based on the loadbalancerrouting and the destinations + } + apiNetLoadBalancer, err := r.APINetInterface.CoreV1alpha1(). LoadBalancers(r.APINetNamespace). Apply(ctx, apiNetLoadBalancerApplyCfg, metav1.ApplyOptions{FieldManager: string(fieldOwner), Force: true}) diff --git a/cmd/apinetlet/main.go b/cmd/apinetlet/main.go index b29afde1..939ea97f 100644 --- a/cmd/apinetlet/main.go +++ b/cmd/apinetlet/main.go @@ -11,6 +11,8 @@ import ( "os" "path/filepath" + flag "github.com/spf13/pflag" + ironcorenetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" apinetletconfig "github.com/ironcore-dev/ironcore-net/apinetlet/client/config" @@ -21,14 +23,10 @@ import ( ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" "github.com/ironcore-dev/ironcore/utils/client/config" - flag "github.com/spf13/pflag" - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" - _ "k8s.io/client-go/plugin/pkg/client/auth" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/certwatcher" @@ -37,7 +35,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log/zap" "sigs.k8s.io/controller-runtime/pkg/metrics/filters" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - //+kubebuilder:scaffold:imports + + _ "k8s.io/client-go/plugin/pkg/client/auth" ) var ( @@ -74,6 +73,9 @@ func main() { var watchNamespace string var watchFilterValue string + + var isNodeAffinityAware bool + var tlsOpts []func(*tls.Config) flag.StringVar(&metricsAddr, "metrics-bind-address", "0", "The address the metrics endpoint binds to. "+ @@ -89,6 +91,7 @@ func main() { flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&isNodeAffinityAware, "is-node-affinity-aware", false, "If set, will determine node affinity topology for loadbalancer daemonsets.") configOptions.BindFlags(flag.CommandLine) apiNetGetConfigOptions.BindFlags(flag.CommandLine, config.WithNamePrefix(apiNetFlagPrefix)) @@ -248,11 +251,12 @@ func main() { } if err = (&controllers.LoadBalancerReconciler{ - Client: mgr.GetClient(), - APINetClient: apiNetCluster.GetClient(), - APINetInterface: apiNetIface, - APINetNamespace: apiNetNamespace, - WatchFilterValue: watchFilterValue, + Client: mgr.GetClient(), + APINetClient: apiNetCluster.GetClient(), + APINetInterface: apiNetIface, + APINetNamespace: apiNetNamespace, + WatchFilterValue: watchFilterValue, + IsNodeAffinityAware: isNodeAffinityAware, }).SetupWithManager(mgr, apiNetCluster.GetCache()); err != nil { setupLog.Error(err, "unable to create controller", "controller", "LoadBalancer") os.Exit(1) From 7047bc06996483282e9edd0484554d8470fb5da6 Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 17:40:39 +0200 Subject: [PATCH 2/9] apply node affinity configuration when node affinity awareness is enabled Signed-off-by: Benjamin Alpert --- .../controllers/loadbalancer_controller.go | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller.go b/apinetlet/controllers/loadbalancer_controller.go index 2c5ec0eb..8d5f7ee9 100644 --- a/apinetlet/controllers/loadbalancer_controller.go +++ b/apinetlet/controllers/loadbalancer_controller.go @@ -13,6 +13,7 @@ import ( "github.com/ironcore-dev/controller-utils/clientutils" apinetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" + corev1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" "github.com/ironcore-dev/ironcore-net/apimachinery/api/net" apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" apinetlethandler "github.com/ironcore-dev/ironcore-net/apinetlet/handler" @@ -153,13 +154,13 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } - _, apiNetDestinations, err := r.prepareApiNetDestinations(ctx, loadBalancer) + apiNetDestinations, _, err := r.prepareApiNetDestinations(ctx, loadBalancer) if err != nil { return ctrl.Result{}, fmt.Errorf("error preparing APINet destinations: %w", err) } log.V(1).Info("Applying APINet load balancer") - apiNetLoadBalancer, err := r.applyAPINetLoadBalancer(ctx, loadBalancer, apiNetNetworkName) + apiNetLoadBalancer, err := r.applyAPINetLoadBalancer(ctx, loadBalancer, apiNetDestinations, apiNetNetworkName) if err != nil { return ctrl.Result{}, fmt.Errorf("error applying APINet load balancer: %w", err) } @@ -181,7 +182,7 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } -func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer) (*networkingv1alpha1.LoadBalancerRouting, []apinetv1alpha1.LoadBalancerDestination, error) { +func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer) ([]apinetv1alpha1.LoadBalancerDestination, *networkingv1alpha1.LoadBalancerRouting, error) { apiNetDsts := make([]apinetv1alpha1.LoadBalancerDestination, 0) loadBalancerRouting := &networkingv1alpha1.LoadBalancerRouting{} if err := r.Get(ctx, client.ObjectKeyFromObject(loadBalancer), loadBalancerRouting); client.IgnoreNotFound(err) != nil { @@ -209,7 +210,7 @@ func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, }) } - return loadBalancerRouting, apiNetDsts, nil + return apiNetDsts, loadBalancerRouting, nil } func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetLoadBalancer *apinetv1alpha1.LoadBalancer, apiNetDsts []apinetv1alpha1.LoadBalancerDestination) error { @@ -287,7 +288,7 @@ func (r *LoadBalancerReconciler) getInternalLoadBalancerAPINetIPs(ctx context.Co return ips, nil } -func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetNetworkName string) (*apinetv1alpha1.LoadBalancer, error) { +func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetDestinations []apinetv1alpha1.LoadBalancerDestination, apiNetNetworkName string) (*apinetv1alpha1.LoadBalancer, error) { apiNetLoadBalancerType, err := loadBalancerTypeToAPINetLoadBalancerType(loadBalancer.Spec.Type) if err != nil { return nil, err @@ -328,7 +329,24 @@ func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, lo ) if r.IsNodeAffinityAware { - // // TODO(balpert): extend the apiNetLoadBalancerApplyCfg with nodeaffinity based on the loadbalancerrouting and the destinations + apiNetDestinationNames := make([]string, 0) + for i := range apiNetDestinations { + apiNetDestinationNames = append(apiNetDestinationNames, apiNetDestinations[i].TargetRef.NodeRef.Name) + } + apiNetLoadBalancerApplyCfg.Spec.Template.Spec.Affinity. + WithNodeAffinity(apinetv1alpha1ac.NodeAffinity(). + WithRequiredDuringSchedulingIgnoredDuringExecution( + apinetv1alpha1ac.NodeSelector(). + WithNodeSelectorTerms( + apinetv1alpha1ac.NodeSelectorTerm(). + WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement(). + WithKey("metadata.name"). + WithOperator(corev1alpha1.NodeSelectorOpIn). + WithValues(apiNetDestinationNames...), + ), + ), + ), + ) } apiNetLoadBalancer, err := r.APINetInterface.CoreV1alpha1(). From c7620cf9bb4014ef8829c02aeb0727a77973f80a Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 17:58:48 +0200 Subject: [PATCH 3/9] remove unused return val Signed-off-by: Benjamin Alpert --- apinetlet/controllers/loadbalancer_controller.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller.go b/apinetlet/controllers/loadbalancer_controller.go index 8d5f7ee9..66e0ef5c 100644 --- a/apinetlet/controllers/loadbalancer_controller.go +++ b/apinetlet/controllers/loadbalancer_controller.go @@ -154,7 +154,7 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } - apiNetDestinations, _, err := r.prepareApiNetDestinations(ctx, loadBalancer) + apiNetDestinations, err := r.prepareApiNetDestinations(ctx, loadBalancer) if err != nil { return ctrl.Result{}, fmt.Errorf("error preparing APINet destinations: %w", err) } @@ -182,11 +182,11 @@ func (r *LoadBalancerReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } -func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer) ([]apinetv1alpha1.LoadBalancerDestination, *networkingv1alpha1.LoadBalancerRouting, error) { +func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer) ([]apinetv1alpha1.LoadBalancerDestination, error) { apiNetDsts := make([]apinetv1alpha1.LoadBalancerDestination, 0) loadBalancerRouting := &networkingv1alpha1.LoadBalancerRouting{} if err := r.Get(ctx, client.ObjectKeyFromObject(loadBalancer), loadBalancerRouting); client.IgnoreNotFound(err) != nil { - return nil, nil, fmt.Errorf("error getting load balancer routing: %w", err) + return nil, fmt.Errorf("error getting load balancer routing: %w", err) } for _, dst := range loadBalancerRouting.Destinations { @@ -210,7 +210,7 @@ func (r *LoadBalancerReconciler) prepareApiNetDestinations(ctx context.Context, }) } - return apiNetDsts, loadBalancerRouting, nil + return apiNetDsts, nil } func (r *LoadBalancerReconciler) manageAPINetLoadBalancerRouting(ctx context.Context, loadBalancer *networkingv1alpha1.LoadBalancer, apiNetLoadBalancer *apinetv1alpha1.LoadBalancer, apiNetDsts []apinetv1alpha1.LoadBalancerDestination) error { From 32a73c4c8a48819e749ddde8a0bd072b3e4192ac Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 21:07:08 +0200 Subject: [PATCH 4/9] fix duplicate import Signed-off-by: Benjamin Alpert --- apinetlet/controllers/loadbalancer_controller.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller.go b/apinetlet/controllers/loadbalancer_controller.go index 66e0ef5c..a41d962d 100644 --- a/apinetlet/controllers/loadbalancer_controller.go +++ b/apinetlet/controllers/loadbalancer_controller.go @@ -13,7 +13,6 @@ import ( "github.com/ironcore-dev/controller-utils/clientutils" apinetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" - corev1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" "github.com/ironcore-dev/ironcore-net/apimachinery/api/net" apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" apinetlethandler "github.com/ironcore-dev/ironcore-net/apinetlet/handler" @@ -341,7 +340,7 @@ func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, lo apinetv1alpha1ac.NodeSelectorTerm(). WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement(). WithKey("metadata.name"). - WithOperator(corev1alpha1.NodeSelectorOpIn). + WithOperator(apinetv1alpha1.NodeSelectorOpIn). WithValues(apiNetDestinationNames...), ), ), From a84f31531593fd9d20e948259bb50a73efe87fd9 Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 21:59:34 +0200 Subject: [PATCH 5/9] enable node awareness in test suite --- .../controllers/controllers_suite_test.go | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/apinetlet/controllers/controllers_suite_test.go b/apinetlet/controllers/controllers_suite_test.go index d29704e6..4504ba35 100644 --- a/apinetlet/controllers/controllers_suite_test.go +++ b/apinetlet/controllers/controllers_suite_test.go @@ -13,17 +13,15 @@ import ( "github.com/ironcore-dev/controller-utils/buildutils" "github.com/ironcore-dev/controller-utils/modutils" - apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" - apinetclient "github.com/ironcore-dev/ironcore-net/internal/client" - apinetv1alpha1 "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" + apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" ironcorenet "github.com/ironcore-dev/ironcore-net/client-go/ironcorenet/versioned" + apinetclient "github.com/ironcore-dev/ironcore-net/internal/client" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" envtestutils "github.com/ironcore-dev/ironcore/utils/envtest" "github.com/ironcore-dev/ironcore/utils/envtest/apiserver" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -32,11 +30,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" "sigs.k8s.io/controller-runtime/pkg/envtest" - . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" //+kubebuilder:scaffold:imports + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to @@ -187,10 +188,11 @@ func SetupTest(apiNetNamespace *corev1.Namespace) *corev1.Namespace { }).SetupWithManager(k8sManager, k8sManager.GetCache())).To(Succeed()) Expect((&LoadBalancerReconciler{ - Client: k8sManager.GetClient(), - APINetClient: k8sManager.GetClient(), - APINetInterface: apiNetInterface, - APINetNamespace: apiNetNamespace.Name, + Client: k8sManager.GetClient(), + APINetClient: k8sManager.GetClient(), + APINetInterface: apiNetInterface, + APINetNamespace: apiNetNamespace.Name, + IsNodeAffinityAware: true, }).SetupWithManager(k8sManager, k8sManager.GetCache())).To(Succeed()) Expect((&NetworkPolicyReconciler{ @@ -206,7 +208,6 @@ func SetupTest(apiNetNamespace *corev1.Namespace) *corev1.Namespace { defer GinkgoRecover() Expect(k8sManager.Start(mgrCtx)).To(Succeed(), "failed to start manager") }() - }) return apiNetNamespace From 56ee27c5e8e378661a33bc7d51ee432c68e143d0 Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 22:04:13 +0200 Subject: [PATCH 6/9] fix pos --- apinetlet/controllers/controllers_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apinetlet/controllers/controllers_suite_test.go b/apinetlet/controllers/controllers_suite_test.go index 4504ba35..f83de1b7 100644 --- a/apinetlet/controllers/controllers_suite_test.go +++ b/apinetlet/controllers/controllers_suite_test.go @@ -33,11 +33,11 @@ import ( logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" - //+kubebuilder:scaffold:imports . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" + //+kubebuilder:scaffold:imports ) // These tests use Ginkgo (BDD-style Go testing framework). Refer to From 71c09f9aa435df42f6216ef808856c889199f043 Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 9 Jun 2025 22:09:47 +0200 Subject: [PATCH 7/9] only apply node affinity when destinations are actually available --- .../controllers/loadbalancer_controller.go | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller.go b/apinetlet/controllers/loadbalancer_controller.go index a41d962d..d26bc36c 100644 --- a/apinetlet/controllers/loadbalancer_controller.go +++ b/apinetlet/controllers/loadbalancer_controller.go @@ -332,20 +332,22 @@ func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, lo for i := range apiNetDestinations { apiNetDestinationNames = append(apiNetDestinationNames, apiNetDestinations[i].TargetRef.NodeRef.Name) } - apiNetLoadBalancerApplyCfg.Spec.Template.Spec.Affinity. - WithNodeAffinity(apinetv1alpha1ac.NodeAffinity(). - WithRequiredDuringSchedulingIgnoredDuringExecution( - apinetv1alpha1ac.NodeSelector(). - WithNodeSelectorTerms( - apinetv1alpha1ac.NodeSelectorTerm(). - WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement(). - WithKey("metadata.name"). - WithOperator(apinetv1alpha1.NodeSelectorOpIn). - WithValues(apiNetDestinationNames...), - ), - ), - ), - ) + if len(apiNetDestinations) > 0 { + apiNetLoadBalancerApplyCfg.Spec.Template.Spec.Affinity. + WithNodeAffinity(apinetv1alpha1ac.NodeAffinity(). + WithRequiredDuringSchedulingIgnoredDuringExecution( + apinetv1alpha1ac.NodeSelector(). + WithNodeSelectorTerms( + apinetv1alpha1ac.NodeSelectorTerm(). + WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement(). + WithKey("metadata.name"). + WithOperator(apinetv1alpha1.NodeSelectorOpIn). + WithValues(apiNetDestinationNames...), + ), + ), + ), + ) + } } apiNetLoadBalancer, err := r.APINetInterface.CoreV1alpha1(). From 73b591a7c1b158788fa2eeaff11ec6b23e5abb6d Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Tue, 10 Jun 2025 10:54:55 +0200 Subject: [PATCH 8/9] create a node selector term for each destination to properly OR the selection --- .../controllers/loadbalancer_controller.go | 35 +++++++++---------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller.go b/apinetlet/controllers/loadbalancer_controller.go index d26bc36c..5fd9c945 100644 --- a/apinetlet/controllers/loadbalancer_controller.go +++ b/apinetlet/controllers/loadbalancer_controller.go @@ -328,25 +328,24 @@ func (r *LoadBalancerReconciler) applyAPINetLoadBalancer(ctx context.Context, lo ) if r.IsNodeAffinityAware { - apiNetDestinationNames := make([]string, 0) - for i := range apiNetDestinations { - apiNetDestinationNames = append(apiNetDestinationNames, apiNetDestinations[i].TargetRef.NodeRef.Name) - } if len(apiNetDestinations) > 0 { - apiNetLoadBalancerApplyCfg.Spec.Template.Spec.Affinity. - WithNodeAffinity(apinetv1alpha1ac.NodeAffinity(). - WithRequiredDuringSchedulingIgnoredDuringExecution( - apinetv1alpha1ac.NodeSelector(). - WithNodeSelectorTerms( - apinetv1alpha1ac.NodeSelectorTerm(). - WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement(). - WithKey("metadata.name"). - WithOperator(apinetv1alpha1.NodeSelectorOpIn). - WithValues(apiNetDestinationNames...), - ), - ), - ), - ) + nodeSelector := apinetv1alpha1ac.NodeSelector() + for i := range apiNetDestinations { + if apiNetDestinations[i].TargetRef != nil { + apiNetDestinationName := apiNetDestinations[i].TargetRef.NodeRef.Name + nodeSelector.WithNodeSelectorTerms(apinetv1alpha1ac.NodeSelectorTerm().WithMatchFields(apinetv1alpha1ac.NodeSelectorRequirement(). + WithKey("metadata.name"). + WithOperator(apinetv1alpha1.NodeSelectorOpIn). + WithValues([]string{apiNetDestinationName}...), + )) + } + } + if len(nodeSelector.NodeSelectorTerms) > 0 { + apiNetLoadBalancerApplyCfg.Spec.Template.Spec.Affinity. + WithNodeAffinity(apinetv1alpha1ac.NodeAffinity(). + WithRequiredDuringSchedulingIgnoredDuringExecution(nodeSelector), + ) + } } } From 3a8a1d3db56e8a01d25de2db2ddbd530a9765c88 Mon Sep 17 00:00:00 2001 From: Benjamin Alpert Date: Mon, 16 Jun 2025 13:43:13 +0200 Subject: [PATCH 9/9] add test for node affinity --- .../loadbalancer_controller_test.go | 124 +++++++++++++++++- 1 file changed, 121 insertions(+), 3 deletions(-) diff --git a/apinetlet/controllers/loadbalancer_controller_test.go b/apinetlet/controllers/loadbalancer_controller_test.go index 68f17715..bab5fdbe 100644 --- a/apinetlet/controllers/loadbalancer_controller_test.go +++ b/apinetlet/controllers/loadbalancer_controller_test.go @@ -4,19 +4,23 @@ package controllers import ( + "net/netip" + "github.com/ironcore-dev/ironcore-net/api/core/v1alpha1" "github.com/ironcore-dev/ironcore-net/apimachinery/api/net" apinetletclient "github.com/ironcore-dev/ironcore-net/apinetlet/client" commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + . "github.com/ironcore-dev/ironcore/utils/testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gstruct" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" ) @@ -251,4 +255,118 @@ var _ = Describe("LoadBalancerController", func() { }))), ) }) + + It("should manage the APINet load balancer and its node affintity", func(ctx SpecContext) { + By("creating a load balancer") + loadBalancer := &networkingv1alpha1.LoadBalancer{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + GenerateName: "load-balancer-", + }, + Spec: networkingv1alpha1.LoadBalancerSpec{ + Type: networkingv1alpha1.LoadBalancerTypePublic, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + NetworkRef: corev1.LocalObjectReference{Name: network.Name}, + }, + } + Expect(k8sClient.Create(ctx, loadBalancer)).To(Succeed()) + DeferCleanup(k8sClient.Delete, loadBalancer) + + By("creating the load balancer routing") + lbRouting := &networkingv1alpha1.LoadBalancerRouting{ + ObjectMeta: metav1.ObjectMeta{ + Name: loadBalancer.Name, + Namespace: loadBalancer.Namespace, + }, + NetworkRef: commonv1alpha1.LocalUIDReference{ + Name: network.Name, + UID: network.UID, + }, + Destinations: []networkingv1alpha1.LoadBalancerDestination{ + { + IP: commonv1alpha1.IP{Addr: netip.MustParseAddr("192.168.0.1")}, + TargetRef: &networkingv1alpha1.LoadBalancerTargetRef{ + UID: "first-nic-uid", + Name: "first-nic-name", + ProviderID: "ironcore-net://namespace/first-apinet-nic-name/first-node-name/first-metalnet-nic-uid", + }, + }, + { + IP: commonv1alpha1.IP{Addr: netip.MustParseAddr("192.168.0.2")}, + TargetRef: &networkingv1alpha1.LoadBalancerTargetRef{ + UID: "second-nic-uid", + Name: "second-nic-name", + ProviderID: "ironcore-net://namespace/second-apinet-nic-name/second-node-name/second-metalnet-nic-uid", + }, + }, + }, + } + Expect(k8sClient.Create(ctx, lbRouting)).To(Succeed()) + DeferCleanup(k8sClient.Delete, lbRouting) + + By("waiting for the APINet load balancer to exist") + apiNetLoadBalancer := &v1alpha1.LoadBalancer{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: apiNetNs.Name, + Name: string(loadBalancer.UID), + }, + } + Eventually(Object(apiNetLoadBalancer)).Should(SatisfyAll( + HaveField("Labels", apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer)), + HaveField("Spec", MatchFields(IgnoreExtras, Fields{ + "Type": Equal(v1alpha1.LoadBalancerTypePublic), + "NetworkRef": Equal(corev1.LocalObjectReference{Name: apiNetNetwork.Name}), + "IPs": ConsistOf(MatchFields(IgnoreExtras, Fields{ + "IPFamily": Equal(corev1.IPv4Protocol), + "Name": Equal("ipv4"), + })), + "Selector": Equal(&metav1.LabelSelector{ + MatchLabels: apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer), + }), + "Template": Equal(v1alpha1.InstanceTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Labels: apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer), + }, + Spec: v1alpha1.InstanceSpec{ + Affinity: &v1alpha1.Affinity{ + InstanceAntiAffinity: &v1alpha1.InstanceAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1alpha1.InstanceAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{ + MatchLabels: apinetletclient.SourceLabels(k8sClient.Scheme(), k8sClient.RESTMapper(), loadBalancer), + }, + TopologyKey: v1alpha1.TopologyZoneLabel, + }, + }, + }, + NodeAffinity: &v1alpha1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &v1alpha1.NodeSelector{ + NodeSelectorTerms: []v1alpha1.NodeSelectorTerm{ + { + MatchFields: []v1alpha1.NodeSelectorRequirement{ + { + Key: "metadata.name", + Operator: v1alpha1.NodeSelectorOpIn, + Values: []string{"first-node-name"}, + }, + }, + }, + { + MatchFields: []v1alpha1.NodeSelectorRequirement{ + { + Key: "metadata.name", + Operator: v1alpha1.NodeSelectorOpIn, + Values: []string{"second-node-name"}, + }, + }, + }, + }, + }, + }, + }, + }, + }), + }))), + ) + }) })