From 18d11d44b21d8801f36765f33168a584c2c1e414 Mon Sep 17 00:00:00 2001 From: Dumitru Ceara Date: Tue, 9 Nov 2021 17:24:56 +0100 Subject: [PATCH] Use Load_Balancer_Groups when supported. This improves scalability of the NB side significantly. Signed-off-by: Dumitru Ceara --- .../ovn/controller/services/load_balancer.go | 32 +++-- .../controller/services/load_balancer_test.go | 15 ++- .../services/services_controller.go | 9 +- go-controller/pkg/ovn/gateway_init.go | 6 + go-controller/pkg/ovn/gateway_test.go | 36 +++++- go-controller/pkg/ovn/libovsdbops/lbgroup.go | 115 ++++++++++++++++++ go-controller/pkg/ovn/libovsdbops/model.go | 10 ++ .../pkg/ovn/loadbalancer/lb_cache.go | 16 +++ .../pkg/ovn/loadbalancer/lb_cache_test.go | 2 + .../pkg/ovn/loadbalancer/loadbalancer.go | 33 ++++- go-controller/pkg/ovn/loadbalancer/types.go | 3 +- go-controller/pkg/ovn/master.go | 37 ++++++ go-controller/pkg/ovn/master_test.go | 47 ++++--- go-controller/pkg/ovn/namespace_test.go | 8 ++ go-controller/pkg/ovn/ovn.go | 7 +- go-controller/pkg/ovn/ovn_test.go | 2 + go-controller/pkg/types/const.go | 2 + 17 files changed, 342 insertions(+), 38 deletions(-) create mode 100644 go-controller/pkg/ovn/libovsdbops/lbgroup.go diff --git a/go-controller/pkg/ovn/controller/services/load_balancer.go b/go-controller/pkg/ovn/controller/services/load_balancer.go index 41ea770ac9..da5c89b8d1 100644 --- a/go-controller/pkg/ovn/controller/services/load_balancer.go +++ b/go-controller/pkg/ovn/controller/services/load_balancer.go @@ -148,16 +148,27 @@ func makeLBName(service *v1.Service, proto v1.Protocol, scope string) string { // It takes a list of (proto:[vips]:port -> [endpoints]) configs and re-aggregates // them to a list of (proto:[vip:port -> [endpoint:port]]) // This load balancer is attached to all node switches. In shared-GW mode, it is also on all routers -func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeInfo) []ovnlb.LB { - nodeSwitches := make([]string, 0, len(nodeInfos)) - nodeRouters := make([]string, 0, len(nodeInfos)) - for _, node := range nodeInfos { - nodeSwitches = append(nodeSwitches, node.switchName) - // For shared gateway, add to the node's GWR as well. - // The node may not have a gateway router - it might be waiting initialization, or - // might have disabled GWR creation via the k8s.ovn.org/l3-gateway-config annotation - if node.gatewayRouterName != "" { - nodeRouters = append(nodeRouters, node.gatewayRouterName) +func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeInfo, useLBGroup bool) []ovnlb.LB { + var nodeSwitches []string + var nodeRouters []string + var groups []string + if useLBGroup { + nodeSwitches = make([]string, 0) + nodeRouters = make([]string, 0) + groups = []string{types.ClusterLBGroupName} + } else { + nodeSwitches = make([]string, 0, len(nodeInfos)) + nodeRouters = make([]string, 0, len(nodeInfos)) + groups = make([]string, 0) + + for _, node := range nodeInfos { + nodeSwitches = append(nodeSwitches, node.switchName) + // For shared gateway, add to the node's GWR as well. + // The node may not have a gateway router - it might be waiting initialization, or + // might have disabled GWR creation via the k8s.ovn.org/l3-gateway-config annotation + if node.gatewayRouterName != "" { + nodeRouters = append(nodeRouters, node.gatewayRouterName) + } } } @@ -177,6 +188,7 @@ func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeIn Switches: nodeSwitches, Routers: nodeRouters, + Groups: groups, } for _, config := range cfgs { diff --git a/go-controller/pkg/ovn/controller/services/load_balancer_test.go b/go-controller/pkg/ovn/controller/services/load_balancer_test.go index ae78dc85db..46682b97fc 100644 --- a/go-controller/pkg/ovn/controller/services/load_balancer_test.go +++ b/go-controller/pkg/ovn/controller/services/load_balancer_test.go @@ -766,8 +766,9 @@ func Test_buildClusterLBs(t *testing.T) { "k8s.ovn.org/owner": fmt.Sprintf("%s/%s", namespace, name), } - defaultRouters := []string{"gr-node-a", "gr-node-b"} - defaultSwitches := []string{"switch-node-a", "switch-node-b"} + defaultRouters := []string{} + defaultSwitches := []string{} + defaultGroups := []string{"clusterLBGroup"} tc := []struct { name string @@ -818,6 +819,7 @@ func Test_buildClusterLBs(t *testing.T) { Routers: defaultRouters, Switches: defaultSwitches, + Groups: defaultGroups, }, }, }, @@ -857,8 +859,9 @@ func Test_buildClusterLBs(t *testing.T) { }, }, - Routers: defaultRouters, Switches: defaultSwitches, + Routers: defaultRouters, + Groups: defaultGroups, }, { Name: fmt.Sprintf("Service_%s/%s_UDP_cluster", namespace, name), @@ -871,8 +874,9 @@ func Test_buildClusterLBs(t *testing.T) { }, }, - Routers: defaultRouters, Switches: defaultSwitches, + Routers: defaultRouters, + Groups: defaultGroups, }, }, }, @@ -928,6 +932,7 @@ func Test_buildClusterLBs(t *testing.T) { Routers: defaultRouters, Switches: defaultSwitches, + Groups: defaultGroups, }, }, }, @@ -935,7 +940,7 @@ func Test_buildClusterLBs(t *testing.T) { for i, tt := range tc { t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) { - actual := buildClusterLBs(tt.service, tt.configs, tt.nodeInfos) + actual := buildClusterLBs(tt.service, tt.configs, tt.nodeInfos, true) assert.Equal(t, tt.expected, actual) }) } diff --git a/go-controller/pkg/ovn/controller/services/services_controller.go b/go-controller/pkg/ovn/controller/services/services_controller.go index fda235a8ba..74545308b6 100644 --- a/go-controller/pkg/ovn/controller/services/services_controller.go +++ b/go-controller/pkg/ovn/controller/services/services_controller.go @@ -148,17 +148,22 @@ type Controller struct { // if a service's config hasn't changed alreadyApplied map[string][]ovnlb.LB alreadyAppliedLock sync.Mutex + + // 'true' if Load_Balancer_Group is supported. + useLBGroups bool } // Run will not return until stopCh is closed. workers determines how many // endpoints will be handled in parallel. -func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair bool) error { +func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGroups bool) error { defer utilruntime.HandleCrash() defer c.queue.ShutDown() klog.Infof("Starting controller %s", controllerName) defer klog.Infof("Shutting down controller %s", controllerName) + c.useLBGroups = useLBGroups + // Wait for the caches to be synced klog.Info("Waiting for informer caches to sync") if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.servicesSynced, c.endpointSlicesSynced, c.nodesSynced) { @@ -296,7 +301,7 @@ func (c *Controller) syncService(key string) error { // Convert the LB configs in to load-balancer objects nodeInfos := c.nodeTracker.allNodes() - clusterLBs := buildClusterLBs(service, clusterConfigs, nodeInfos) + clusterLBs := buildClusterLBs(service, clusterConfigs, nodeInfos, c.useLBGroups) perNodeLBs := buildPerNodeLBs(service, perNodeConfigs, nodeInfos) klog.V(5).Infof("Built service %s cluster-wide LB %#v", key, clusterLBs) klog.V(5).Infof("Built service %s per-node LB %#v", key, perNodeLBs) diff --git a/go-controller/pkg/ovn/gateway_init.go b/go-controller/pkg/ovn/gateway_init.go index 954055867a..39d0f90c2a 100644 --- a/go-controller/pkg/ovn/gateway_init.go +++ b/go-controller/pkg/ovn/gateway_init.go @@ -50,6 +50,11 @@ func (oc *Controller) gatewayInit(nodeName string, clusterIPSubnet []*net.IPNet, Options: logicalRouterOptions, ExternalIDs: logicalRouterExternalIDs, } + + if oc.loadBalancerGroupUUID != "" { + logicalRouter.LoadBalancerGroup = []string{oc.loadBalancerGroupUUID} + } + opModels := []libovsdbops.OperationModel{ { Model: &logicalRouter, @@ -57,6 +62,7 @@ func (oc *Controller) gatewayInit(nodeName string, clusterIPSubnet []*net.IPNet, OnModelUpdates: []interface{}{ &logicalRouter.Options, &logicalRouter.ExternalIDs, + &logicalRouter.LoadBalancerGroup, }, }, } diff --git a/go-controller/pkg/ovn/gateway_test.go b/go-controller/pkg/ovn/gateway_test.go index f1bb8f3152..1dc7f00e89 100644 --- a/go-controller/pkg/ovn/gateway_test.go +++ b/go-controller/pkg/ovn/gateway_test.go @@ -160,9 +160,10 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus "physical_ip": physicalIPs[0], "physical_ips": strings.Join(physicalIPs, ","), }, - Ports: []string{gwRouterPort + "-UUID", externalRouterPort + "-UUID"}, - StaticRoutes: grStaticRoutes, - Nat: natUUIDs, + Ports: []string{gwRouterPort + "-UUID", externalRouterPort + "-UUID"}, + StaticRoutes: grStaticRoutes, + Nat: natUUIDs, + LoadBalancerGroup: []string{types.ClusterLBGroupName + "-UUID"}, }) testData = append(testData, expectedOVNClusterRouter) @@ -218,6 +219,10 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus UUID: externalSwitch + "-UUID", Name: externalSwitch, Ports: []string{l3GatewayConfig.InterfaceID + "-UUID", externalSwitchPortToRouter + "-UUID"}, + }, + &nbdb.LoadBalancerGroup{ + Name: types.ClusterLBGroupName, + UUID: types.ClusterLBGroupName + "-UUID", }) return testData } @@ -254,6 +259,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: nodeName + "-UUID", Name: nodeName, } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterLBGroupName + "-UUID", + Name: types.ClusterLBGroupName, + } fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ &nbdb.LogicalSwitch{ @@ -262,6 +271,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { }, expectedOVNClusterRouter, expectedNodeSwitch, + expectedClusterLBGroup, }, }) @@ -300,6 +310,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: nodeName + "-UUID", Name: nodeName, } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterLBGroupName + "-UUID", + Name: types.ClusterLBGroupName, + } fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ &nbdb.LogicalSwitch{ @@ -308,6 +322,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { }, expectedOVNClusterRouter, expectedNodeSwitch, + expectedClusterLBGroup, }, }) @@ -347,6 +362,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: nodeName + "-UUID", Name: nodeName, } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterLBGroupName + "-UUID", + Name: types.ClusterLBGroupName, + } fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ &nbdb.LogicalSwitch{ @@ -355,6 +374,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { }, expectedOVNClusterRouter, expectedNodeSwitch, + expectedClusterLBGroup, }, }) @@ -394,6 +414,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: nodeName + "-UUID", Name: nodeName, } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + UUID: types.ClusterLBGroupName + "-UUID", + Name: types.ClusterLBGroupName, + } fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ &nbdb.LogicalSwitch{ @@ -402,6 +426,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { }, expectedOVNClusterRouter, expectedNodeSwitch, + expectedClusterLBGroup, }, }) @@ -450,6 +475,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: nodeName + "-UUID", Name: nodeName, } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + Name: types.ClusterLBGroupName, + UUID: types.ClusterLBGroupName + "-UUID", + } fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{ NBData: []libovsdbtest.TestData{ &nbdb.LogicalSwitch{ @@ -458,6 +487,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { }, expectedOVNClusterRouter, expectedNodeSwitch, + expectedClusterLBGroup, }, }) diff --git a/go-controller/pkg/ovn/libovsdbops/lbgroup.go b/go-controller/pkg/ovn/libovsdbops/lbgroup.go new file mode 100644 index 0000000000..a42ea2b798 --- /dev/null +++ b/go-controller/pkg/ovn/libovsdbops/lbgroup.go @@ -0,0 +1,115 @@ +package libovsdbops + +import ( + "context" + "fmt" + + libovsdbclient "github.com/ovn-org/libovsdb/client" + "github.com/ovn-org/libovsdb/model" + libovsdb "github.com/ovn-org/libovsdb/ovsdb" + + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" +) + +// findLBGroup looks up the Group in the cache and sets the UUID +func findLBGroup(nbClient libovsdbclient.Client, group *nbdb.LoadBalancerGroup) error { + if group.UUID != "" && !IsNamedUUID(group.UUID) { + return nil + } + + ctx, cancel := context.WithTimeout(context.Background(), types.OVSDBTimeout) + defer cancel() + groups := []nbdb.LoadBalancerGroup{} + err := nbClient.WhereCache(func(item *nbdb.LoadBalancerGroup) bool { + return item.Name == group.Name + }).List(ctx, &groups) + if err != nil { + return fmt.Errorf("can't find LB group %+v: %v", *group, err) + } + + if len(groups) > 1 { + return fmt.Errorf("unexpectedly found multiple LB Groups: %+v", groups) + } + + if len(groups) == 0 { + return libovsdbclient.ErrNotFound + } + + group.UUID = groups[0].UUID + return nil +} + +func AddLoadBalancersToGroupOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, group *nbdb.LoadBalancerGroup, lbs ...*nbdb.LoadBalancer) ([]libovsdb.Operation, error) { + if ops == nil { + ops = []libovsdb.Operation{} + } + if len(lbs) == 0 { + return ops, nil + } + + err := findLBGroup(nbClient, group) + if err != nil { + return nil, err + } + + lbUUIDs := make([]string, 0, len(lbs)) + for _, lb := range lbs { + lbUUIDs = append(lbUUIDs, lb.UUID) + } + + op, err := nbClient.Where(group).Mutate(group, model.Mutation{ + Field: &group.LoadBalancer, + Mutator: libovsdb.MutateOperationInsert, + Value: lbUUIDs, + }) + if err != nil { + return nil, err + } + ops = append(ops, op...) + return ops, nil +} + +func RemoveLoadBalancersFromGroupOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, group *nbdb.LoadBalancerGroup, lbs ...*nbdb.LoadBalancer) ([]libovsdb.Operation, error) { + if ops == nil { + ops = []libovsdb.Operation{} + } + if len(lbs) == 0 { + return ops, nil + } + + err := findLBGroup(nbClient, group) + if err != nil { + return nil, err + } + + lbUUIDs := make([]string, 0, len(lbs)) + for _, lb := range lbs { + lbUUIDs = append(lbUUIDs, lb.UUID) + } + + op, err := nbClient.Where(group).Mutate(group, model.Mutation{ + Field: &group.LoadBalancer, + Mutator: libovsdb.MutateOperationDelete, + Value: lbUUIDs, + }) + if err != nil { + return nil, err + } + ops = append(ops, op...) + + return ops, nil +} + +func ListGroupsWithLoadBalancers(nbClient libovsdbclient.Client) ([]nbdb.LoadBalancerGroup, error) { + groups := &[]nbdb.LoadBalancerGroup{} + ctx, cancel := context.WithTimeout(context.Background(), types.OVSDBTimeout) + defer cancel() + err := nbClient.WhereCache(func(item *nbdb.LoadBalancerGroup) bool { + return item.LoadBalancer != nil + }).List(ctx, groups) + if err != nil { + return nil, err + } + return *groups, nil +} diff --git a/go-controller/pkg/ovn/libovsdbops/model.go b/go-controller/pkg/ovn/libovsdbops/model.go index 89ed3ff28c..6633ee4e4b 100644 --- a/go-controller/pkg/ovn/libovsdbops/model.go +++ b/go-controller/pkg/ovn/libovsdbops/model.go @@ -21,6 +21,8 @@ func getUUID(model model.Model) string { return t.UUID case *nbdb.LoadBalancer: return t.UUID + case *nbdb.LoadBalancerGroup: + return t.UUID case *nbdb.LogicalRouter: return t.UUID case *nbdb.LogicalRouterPolicy: @@ -60,6 +62,8 @@ func setUUID(model model.Model, uuid string) { t.UUID = uuid case *nbdb.LoadBalancer: t.UUID = uuid + case *nbdb.LoadBalancerGroup: + t.UUID = uuid case *nbdb.LogicalRouter: t.UUID = uuid case *nbdb.LogicalRouterPolicy: @@ -113,6 +117,10 @@ func copyIndexes(model model.Model) model.Model { return &nbdb.LoadBalancer{ UUID: t.UUID, } + case *nbdb.LoadBalancerGroup: + return &nbdb.LoadBalancerGroup{ + UUID: t.UUID, + } case *nbdb.LogicalRouter: return &nbdb.LogicalRouter{ UUID: t.UUID, @@ -179,6 +187,8 @@ func getListFromModel(model model.Model) interface{} { return &[]nbdb.GatewayChassis{} case *nbdb.LoadBalancer: return &[]nbdb.LoadBalancer{} + case *nbdb.LoadBalancerGroup: + return &[]nbdb.LoadBalancerGroup{} case *nbdb.LogicalRouter: return &[]nbdb.LogicalRouter{} case *nbdb.LogicalRouterPolicy: diff --git a/go-controller/pkg/ovn/loadbalancer/lb_cache.go b/go-controller/pkg/ovn/loadbalancer/lb_cache.go index 5675b8e0c9..3e0caadd63 100644 --- a/go-controller/pkg/ovn/loadbalancer/lb_cache.go +++ b/go-controller/pkg/ovn/loadbalancer/lb_cache.go @@ -49,6 +49,7 @@ type CachedLB struct { Switches sets.String Routers sets.String + Groups sets.String } // update the database with any existing LBs, along with any @@ -73,6 +74,7 @@ func (c *LBCache) update(existing []LB, toDelete []string) { Switches: sets.NewString(lb.Switches...), Routers: sets.NewString(lb.Routers...), + Groups: sets.NewString(lb.Groups...), } } } @@ -192,6 +194,19 @@ func newCache(nbClient libovsdbclient.Client) (*LBCache, error) { } } + groups, err := libovsdbops.ListGroupsWithLoadBalancers(nbClient) + if err != nil { + return nil, err + } + + for _, group := range groups { + for _, lbuuid := range group.LoadBalancer { + if lb, ok := c.existing[lbuuid]; ok { + lb.Groups.Insert(group.Name) + } + } + } + return &c, nil } @@ -211,6 +226,7 @@ func listLBs(nbClient libovsdbclient.Client) ([]CachedLB, error) { VIPs: sets.String{}, Switches: sets.String{}, Routers: sets.String{}, + Groups: sets.String{}, } if lb.Protocol != nil { diff --git a/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go b/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go index e75d547797..5de27c1c74 100644 --- a/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go +++ b/go-controller/pkg/ovn/loadbalancer/lb_cache_test.go @@ -95,6 +95,7 @@ func TestNewCache(t *testing.T) { Routers: sets.String{ "GR_ovn-control-plane": {}, }, + Groups: sets.String{}, }, "7dc190c4-c615-467f-af83-9856d832c9a0": { UUID: "7dc190c4-c615-467f-af83-9856d832c9a0", @@ -113,6 +114,7 @@ func TestNewCache(t *testing.T) { "GR_ovn-worker": {}, "GR_ovn-worker2": {}, }, + Groups: sets.String{}, }, }, c.existing) diff --git a/go-controller/pkg/ovn/loadbalancer/loadbalancer.go b/go-controller/pkg/ovn/loadbalancer/loadbalancer.go index 39d6c527cc..1063833e04 100644 --- a/go-controller/pkg/ovn/loadbalancer/loadbalancer.go +++ b/go-controller/pkg/ovn/loadbalancer/loadbalancer.go @@ -55,6 +55,8 @@ func EnsureLBs(nbClient libovsdbclient.Client, externalIDs map[string]string, LB removeLBsFromSwitch := map[string][]*nbdb.LoadBalancer{} addLBsToRouter := map[string][]*nbdb.LoadBalancer{} removesLBsFromRouter := map[string][]*nbdb.LoadBalancer{} + addLBsToGroups := map[string][]*nbdb.LoadBalancer{} + removeLBsFromGroups := map[string][]*nbdb.LoadBalancer{} wantedByName := make(map[string]*LB, len(LBs)) for i, lb := range LBs { wantedByName[lb.Name] = &LBs[i] @@ -63,17 +65,22 @@ func EnsureLBs(nbClient libovsdbclient.Client, externalIDs map[string]string, LB existingLB := existingByName[lb.Name] existingRouters := sets.String{} existingSwitches := sets.String{} + existingGroups := sets.String{} if existingLB != nil { toDelete.Delete(existingLB.UUID) existingRouters = existingLB.Routers existingSwitches = existingLB.Switches + existingGroups = existingLB.Groups } wantRouters := sets.NewString(lb.Routers...) wantSwitches := sets.NewString(lb.Switches...) + wantGroups := sets.NewString(lb.Groups...) mapLBDifferenceByKey(addLBsToSwitch, wantSwitches, existingSwitches, blb) mapLBDifferenceByKey(removeLBsFromSwitch, existingSwitches, wantSwitches, blb) mapLBDifferenceByKey(addLBsToRouter, wantRouters, existingRouters, blb) mapLBDifferenceByKey(removesLBsFromRouter, existingRouters, wantRouters, blb) + mapLBDifferenceByKey(addLBsToGroups, wantGroups, existingGroups, blb) + mapLBDifferenceByKey(removeLBsFromGroups, existingGroups, wantGroups, blb) } ops, err := libovsdbops.CreateOrUpdateLoadBalancersOps(nbClient, nil, lbs...) @@ -129,6 +136,30 @@ func EnsureLBs(nbClient libovsdbclient.Client, externalIDs map[string]string, LB } } + // cache groups for this round of ops + groups := map[string]*nbdb.LoadBalancerGroup{} + getGroup := func(name string) *nbdb.LoadBalancerGroup { + var group *nbdb.LoadBalancerGroup + var found bool + if group, found = groups[name]; !found { + group = &nbdb.LoadBalancerGroup{Name: name} + groups[name] = group + } + return group + } + for k, v := range addLBsToGroups { + ops, err = libovsdbops.AddLoadBalancersToGroupOps(nbClient, ops, getGroup(k), v...) + if err != nil { + return err + } + } + for k, v := range removeLBsFromGroups { + ops, err = libovsdbops.RemoveLoadBalancersFromGroupOps(nbClient, ops, getGroup(k), v...) + if err != nil { + return err + } + } + deleteLBs := make([]*nbdb.LoadBalancer, 0, len(toDelete)) for uuid := range toDelete { deleteLBs = append(deleteLBs, &nbdb.LoadBalancer{UUID: uuid}) @@ -235,7 +266,7 @@ func buildVipMap(rules []LBRule) map[string]string { } // DeleteLBs deletes all load balancer uuids supplied -// Note: this also automatically removes them from the switches and the routers :-) +// Note: this also automatically removes them from the switches, routers, and the groups :-) func DeleteLBs(nbClient libovsdbclient.Client, uuids []string) error { if len(uuids) == 0 { return nil diff --git a/go-controller/pkg/ovn/loadbalancer/types.go b/go-controller/pkg/ovn/loadbalancer/types.go index 540f7e1f89..6464184aad 100644 --- a/go-controller/pkg/ovn/loadbalancer/types.go +++ b/go-controller/pkg/ovn/loadbalancer/types.go @@ -12,9 +12,10 @@ type LB struct { Rules []LBRule - // the names of logical switches and routers that this LB should be attached to + // the names of logical switches, routers and LB groups that this LB should be attached to Switches []string Routers []string + Groups []string } type LBOpts struct { diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index e41c80af5c..de92f1fc8e 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -287,6 +287,38 @@ func (oc *Controller) StartClusterMaster(masterNodeName string) error { } } + // FIXME: When https://github.com/ovn-org/libovsdb/issues/235 is fixed, + // use IsTableSupported(nbdb.LoadBalancerGroup). + if _, _, err := util.RunOVNNbctl("--columns=_uuid", "list", "Load_Balancer_Group"); err != nil { + klog.Warningf("Load Balancer Group support enabled, however version of OVN in use does not support Load Balancer Groups.") + } else { + loadBalancerGroup := nbdb.LoadBalancerGroup{ + Name: types.ClusterLBGroupName, + } + loadBalancerGroupRes := []nbdb.LoadBalancerGroup{} + opModels := []libovsdbops.OperationModel{ + { + Model: &loadBalancerGroup, + ModelPredicate: func(lbg *nbdb.LoadBalancerGroup) bool { return lbg.Name == types.ClusterLBGroupName }, + OnModelUpdates: []interface{}{ + &loadBalancerGroup.Name, + }, + ExistingResult: &loadBalancerGroupRes, + DoAfter: func() { + if len(loadBalancerGroupRes) > 0 { + loadBalancerGroup.UUID = loadBalancerGroupRes[0].UUID + } + }, + ErrNotFound: false, + }, + } + if _, err = oc.modelClient.CreateOrUpdate(opModels...); err != nil { + klog.Errorf("Error creating cluster-wide load balancer group (%v)", err) + return err + } + oc.loadBalancerGroupUUID = loadBalancerGroup.UUID + } + if err := oc.SetupMaster(masterNodeName, nodeNames); err != nil { klog.Errorf("Failed to setup master (%v)", err) return err @@ -794,6 +826,10 @@ func (oc *Controller) ensureNodeLogicalNetwork(node *kapi.Node, hostSubnets []*n } } + if oc.loadBalancerGroupUUID != "" { + logicalSwitch.LoadBalancerGroup = []string{oc.loadBalancerGroupUUID} + } + logicalRouterPortName := types.RouterToSwitchPrefix + nodeName logicalRouterPort := nbdb.LogicalRouterPort{ Name: logicalRouterPortName, @@ -825,6 +861,7 @@ func (oc *Controller) ensureNodeLogicalNetwork(node *kapi.Node, hostSubnets []*n ModelPredicate: func(ls *nbdb.LogicalSwitch) bool { return ls.Name == nodeName }, OnModelUpdates: []interface{}{ &logicalSwitch.OtherConfig, + &logicalSwitch.LoadBalancerGroup, }, }, } diff --git a/go-controller/pkg/ovn/master_test.go b/go-controller/pkg/ovn/master_test.go index 5f12018a03..0858bd28d1 100644 --- a/go-controller/pkg/ovn/master_test.go +++ b/go-controller/pkg/ovn/master_test.go @@ -933,14 +933,19 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + Name: types.ClusterLBGroupName, + UUID: types.ClusterLBGroupName + "-UUID", + } expectedOVNClusterRouter := &nbdb.LogicalRouter{ UUID: types.OVNClusterRouter + "-UUID", Name: types.OVNClusterRouter, } expectedNodeSwitch := &nbdb.LogicalSwitch{ - UUID: node1.Name + "-UUID", - Name: node1.Name, - OtherConfig: map[string]string{"subnet": node1.NodeSubnet}, + UUID: node1.Name + "-UUID", + Name: node1.Name, + OtherConfig: map[string]string{"subnet": node1.NodeSubnet}, + LoadBalancerGroup: []string{expectedClusterLBGroup.UUID}, } expectedClusterRouterPortGroup := &nbdb.PortGroup{ UUID: types.ClusterRtrPortGroupName + "-UUID", @@ -962,13 +967,11 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.OVNJoinSwitch + "-UUID", Name: types.OVNJoinSwitch, }, - &nbdb.LogicalSwitch{ - UUID: node1.Name + "-UUID", - Name: node1.Name, - }, + expectedNodeSwitch, expectedOVNClusterRouter, expectedClusterRouterPortGroup, expectedClusterPortGroup, + expectedClusterLBGroup, }, } var libovsdbOvnNBClient, libovsdbOvnSBClient libovsdbclient.Client @@ -981,6 +984,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { clusterController := NewOvnController(fakeClient, f, stopChan, addressset.NewFakeAddressSetFactory(), libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(0)) + clusterController.loadBalancerGroupUUID = expectedClusterLBGroup.UUID gomega.Expect(clusterController).NotTo(gomega.BeNil()) clusterController.SCTPSupport = true @@ -1124,14 +1128,19 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { err = f.Start() gomega.Expect(err).NotTo(gomega.HaveOccurred()) + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + Name: types.ClusterLBGroupName, + UUID: types.ClusterLBGroupName + "-UUID", + } expectedOVNClusterRouter := &nbdb.LogicalRouter{ UUID: types.OVNClusterRouter + "-UUID", Name: types.OVNClusterRouter, } expectedNodeSwitch := &nbdb.LogicalSwitch{ - UUID: node1.Name + "-UUID", - Name: node1.Name, - OtherConfig: map[string]string{"subnet": node1.NodeSubnet}, + UUID: node1.Name + "-UUID", + Name: node1.Name, + OtherConfig: map[string]string{"subnet": node1.NodeSubnet}, + LoadBalancerGroup: []string{expectedClusterLBGroup.UUID}, } expectedClusterRouterPortGroup := &nbdb.PortGroup{ UUID: types.ClusterRtrPortGroupName + "-UUID", @@ -1153,13 +1162,11 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { UUID: types.OVNJoinSwitch + "-UUID", Name: types.OVNJoinSwitch, }, - &nbdb.LogicalSwitch{ - UUID: node1.Name + "-UUID", - Name: node1.Name, - }, + expectedNodeSwitch, expectedOVNClusterRouter, expectedClusterRouterPortGroup, expectedClusterPortGroup, + expectedClusterLBGroup, }, } var libovsdbOvnNBClient, libovsdbOvnSBClient libovsdbclient.Client @@ -1172,6 +1179,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() { clusterController := NewOvnController(fakeClient, f, stopChan, addressset.NewFakeAddressSetFactory(), libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(0)) + clusterController.loadBalancerGroupUUID = expectedClusterLBGroup.UUID gomega.Expect(clusterController).NotTo(gomega.BeNil()) clusterController.SCTPSupport = true @@ -1448,7 +1456,15 @@ func TestController_allocateNodeSubnets(t *testing.T) { t.Fatalf("Error starting master watch factory: %v", err) } - dbSetup := libovsdbtest.TestSetup{} + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + Name: types.ClusterLBGroupName, + UUID: types.ClusterLBGroupName + "-UUID", + } + dbSetup := libovsdbtest.TestSetup{ + NBData: []libovsdbtest.TestData{ + expectedClusterLBGroup, + }, + } libovsdbOvnNBClient, libovsdbOvnSBClient, libovsdbCleanup, err := libovsdbtest.NewNBSBTestHarness(dbSetup) if err != nil { t.Fatalf("Error creating libovsdb test harness %v", err) @@ -1458,6 +1474,7 @@ func TestController_allocateNodeSubnets(t *testing.T) { clusterController := NewOvnController(fakeClient, f, stopChan, addressset.NewFakeAddressSetFactory(), libovsdbOvnNBClient, libovsdbOvnSBClient, record.NewFakeRecorder(0)) + clusterController.loadBalancerGroupUUID = expectedClusterLBGroup.UUID // configure the cluster allocators for _, subnetString := range tt.networkRanges { diff --git a/go-controller/pkg/ovn/namespace_test.go b/go-controller/pkg/ovn/namespace_test.go index d03ef94415..e2d4679701 100644 --- a/go-controller/pkg/ovn/namespace_test.go +++ b/go-controller/pkg/ovn/namespace_test.go @@ -202,6 +202,10 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { "name": ovntypes.ClusterPortGroupName, }, } + expectedClusterLBGroup := &nbdb.LoadBalancerGroup{ + Name: ovntypes.ClusterLBGroupName, + UUID: ovntypes.ClusterLBGroupName + "-UUID", + } fakeOvn.startWithDBSetup( libovsdbtest.TestSetup{ @@ -214,6 +218,7 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { expectedNodeSwitch, expectedClusterRouterPortGroup, expectedClusterPortGroup, + expectedClusterLBGroup, }, }, &v1.NamespaceList{ @@ -267,6 +272,9 @@ var _ = ginkgo.Describe("OVN Namespace Operations", func() { // Add subnet to otherconfig for node expectedNodeSwitch.OtherConfig = map[string]string{"subnet": node1.NodeSubnet} + // Add cluster LB Group to node switch. + expectedNodeSwitch.LoadBalancerGroup = []string{expectedClusterLBGroup.UUID} + expectedDatabaseState = addNodeLogicalFlows(expectedDatabaseState, expectedOVNClusterRouter, expectedNodeSwitch, expectedClusterRouterPortGroup, expectedClusterPortGroup, &node1, clusterCIDR, config.IPv6Mode) fakeOvn.controller.joinSwIPManager, _ = lsm.NewJoinLogicalSwitchIPManager(fakeOvn.nbClient, []string{node1.Name}) diff --git a/go-controller/pkg/ovn/ovn.go b/go-controller/pkg/ovn/ovn.go index 2ab7e0494f..d20068bbfb 100644 --- a/go-controller/pkg/ovn/ovn.go +++ b/go-controller/pkg/ovn/ovn.go @@ -155,6 +155,9 @@ type Controller struct { // Supports multicast? multicastSupport bool + // Cluster wide Load_Balancer_Group UUID. + loadBalancerGroupUUID string + // Controller used for programming OVN for egress IP eIPC egressIPController @@ -272,6 +275,7 @@ func NewOvnController(ovnClient *util.OVNClientset, wf *factory.WatchFactory, st }, loadbalancerClusterCache: make(map[kapi.Protocol]string), multicastSupport: config.EnableMulticast, + loadBalancerGroupUUID: "", aclLoggingEnabled: true, joinSwIPManager: nil, retryPods: make(map[types.UID]*retryEntry), @@ -1292,9 +1296,10 @@ func (oc *Controller) StartServiceController(wg *sync.WaitGroup, runRepair bool) wg.Add(1) go func() { defer wg.Done() + useLBGroups := oc.loadBalancerGroupUUID != "" // use 5 workers like most of the kubernetes controllers in the // kubernetes controller-manager - err := oc.svcController.Run(5, oc.stopChan, runRepair) + err := oc.svcController.Run(5, oc.stopChan, runRepair, useLBGroups) if err != nil { klog.Errorf("Error running OVN Kubernetes Services controller: %v", err) } diff --git a/go-controller/pkg/ovn/ovn_test.go b/go-controller/pkg/ovn/ovn_test.go index 3237b88a6c..d9c9bd580e 100644 --- a/go-controller/pkg/ovn/ovn_test.go +++ b/go-controller/pkg/ovn/ovn_test.go @@ -7,6 +7,7 @@ import ( addressset "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/address_set" ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing" libovsdbtest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing/libovsdb" + "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types" util "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util" "k8s.io/apimachinery/pkg/runtime" @@ -103,4 +104,5 @@ func (o *FakeOVN) init() { o.nbClient, o.sbClient, o.fakeRecorder) o.controller.multicastSupport = true + o.controller.loadBalancerGroupUUID = types.ClusterLBGroupName + "-UUID" } diff --git a/go-controller/pkg/types/const.go b/go-controller/pkg/types/const.go index cdd46da819..43be09b355 100644 --- a/go-controller/pkg/types/const.go +++ b/go-controller/pkg/types/const.go @@ -126,4 +126,6 @@ const ( ClusterRtrPortGroupName = "clusterRtrPortGroup" OVSDBTimeout = 10 * time.Second + + ClusterLBGroupName = "clusterLBGroup" )