Skip to content

Commit

Permalink
Use Load_Balancer_Groups when supported.
Browse files Browse the repository at this point in the history
This improves scalability of the NB side significantly.

Signed-off-by: Dumitru Ceara <dceara@redhat.com>
  • Loading branch information
dceara committed Dec 7, 2021
1 parent f895b42 commit 18d11d4
Show file tree
Hide file tree
Showing 17 changed files with 342 additions and 38 deletions.
32 changes: 22 additions & 10 deletions go-controller/pkg/ovn/controller/services/load_balancer.go
Expand Up @@ -148,16 +148,27 @@ func makeLBName(service *v1.Service, proto v1.Protocol, scope string) string {
// It takes a list of (proto:[vips]:port -> [endpoints]) configs and re-aggregates
// them to a list of (proto:[vip:port -> [endpoint:port]])
// This load balancer is attached to all node switches. In shared-GW mode, it is also on all routers
func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeInfo) []ovnlb.LB {
nodeSwitches := make([]string, 0, len(nodeInfos))
nodeRouters := make([]string, 0, len(nodeInfos))
for _, node := range nodeInfos {
nodeSwitches = append(nodeSwitches, node.switchName)
// For shared gateway, add to the node's GWR as well.
// The node may not have a gateway router - it might be waiting initialization, or
// might have disabled GWR creation via the k8s.ovn.org/l3-gateway-config annotation
if node.gatewayRouterName != "" {
nodeRouters = append(nodeRouters, node.gatewayRouterName)
func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeInfo, useLBGroup bool) []ovnlb.LB {
var nodeSwitches []string
var nodeRouters []string
var groups []string
if useLBGroup {
nodeSwitches = make([]string, 0)
nodeRouters = make([]string, 0)
groups = []string{types.ClusterLBGroupName}
} else {
nodeSwitches = make([]string, 0, len(nodeInfos))
nodeRouters = make([]string, 0, len(nodeInfos))
groups = make([]string, 0)

for _, node := range nodeInfos {
nodeSwitches = append(nodeSwitches, node.switchName)
// For shared gateway, add to the node's GWR as well.
// The node may not have a gateway router - it might be waiting initialization, or
// might have disabled GWR creation via the k8s.ovn.org/l3-gateway-config annotation
if node.gatewayRouterName != "" {
nodeRouters = append(nodeRouters, node.gatewayRouterName)
}
}
}

Expand All @@ -177,6 +188,7 @@ func buildClusterLBs(service *v1.Service, configs []lbConfig, nodeInfos []nodeIn

Switches: nodeSwitches,
Routers: nodeRouters,
Groups: groups,
}

for _, config := range cfgs {
Expand Down
15 changes: 10 additions & 5 deletions go-controller/pkg/ovn/controller/services/load_balancer_test.go
Expand Up @@ -766,8 +766,9 @@ func Test_buildClusterLBs(t *testing.T) {
"k8s.ovn.org/owner": fmt.Sprintf("%s/%s", namespace, name),
}

defaultRouters := []string{"gr-node-a", "gr-node-b"}
defaultSwitches := []string{"switch-node-a", "switch-node-b"}
defaultRouters := []string{}
defaultSwitches := []string{}
defaultGroups := []string{"clusterLBGroup"}

tc := []struct {
name string
Expand Down Expand Up @@ -818,6 +819,7 @@ func Test_buildClusterLBs(t *testing.T) {

Routers: defaultRouters,
Switches: defaultSwitches,
Groups: defaultGroups,
},
},
},
Expand Down Expand Up @@ -857,8 +859,9 @@ func Test_buildClusterLBs(t *testing.T) {
},
},

Routers: defaultRouters,
Switches: defaultSwitches,
Routers: defaultRouters,
Groups: defaultGroups,
},
{
Name: fmt.Sprintf("Service_%s/%s_UDP_cluster", namespace, name),
Expand All @@ -871,8 +874,9 @@ func Test_buildClusterLBs(t *testing.T) {
},
},

Routers: defaultRouters,
Switches: defaultSwitches,
Routers: defaultRouters,
Groups: defaultGroups,
},
},
},
Expand Down Expand Up @@ -928,14 +932,15 @@ func Test_buildClusterLBs(t *testing.T) {

Routers: defaultRouters,
Switches: defaultSwitches,
Groups: defaultGroups,
},
},
},
}

for i, tt := range tc {
t.Run(fmt.Sprintf("%d_%s", i, tt.name), func(t *testing.T) {
actual := buildClusterLBs(tt.service, tt.configs, tt.nodeInfos)
actual := buildClusterLBs(tt.service, tt.configs, tt.nodeInfos, true)
assert.Equal(t, tt.expected, actual)
})
}
Expand Down
Expand Up @@ -148,17 +148,22 @@ type Controller struct {
// if a service's config hasn't changed
alreadyApplied map[string][]ovnlb.LB
alreadyAppliedLock sync.Mutex

// 'true' if Load_Balancer_Group is supported.
useLBGroups bool
}

// Run will not return until stopCh is closed. workers determines how many
// endpoints will be handled in parallel.
func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair bool) error {
func (c *Controller) Run(workers int, stopCh <-chan struct{}, runRepair, useLBGroups bool) error {
defer utilruntime.HandleCrash()
defer c.queue.ShutDown()

klog.Infof("Starting controller %s", controllerName)
defer klog.Infof("Shutting down controller %s", controllerName)

c.useLBGroups = useLBGroups

// Wait for the caches to be synced
klog.Info("Waiting for informer caches to sync")
if !cache.WaitForNamedCacheSync(controllerName, stopCh, c.servicesSynced, c.endpointSlicesSynced, c.nodesSynced) {
Expand Down Expand Up @@ -296,7 +301,7 @@ func (c *Controller) syncService(key string) error {

// Convert the LB configs in to load-balancer objects
nodeInfos := c.nodeTracker.allNodes()
clusterLBs := buildClusterLBs(service, clusterConfigs, nodeInfos)
clusterLBs := buildClusterLBs(service, clusterConfigs, nodeInfos, c.useLBGroups)
perNodeLBs := buildPerNodeLBs(service, perNodeConfigs, nodeInfos)
klog.V(5).Infof("Built service %s cluster-wide LB %#v", key, clusterLBs)
klog.V(5).Infof("Built service %s per-node LB %#v", key, perNodeLBs)
Expand Down
6 changes: 6 additions & 0 deletions go-controller/pkg/ovn/gateway_init.go
Expand Up @@ -50,13 +50,19 @@ func (oc *Controller) gatewayInit(nodeName string, clusterIPSubnet []*net.IPNet,
Options: logicalRouterOptions,
ExternalIDs: logicalRouterExternalIDs,
}

if oc.loadBalancerGroupUUID != "" {
logicalRouter.LoadBalancerGroup = []string{oc.loadBalancerGroupUUID}
}

opModels := []libovsdbops.OperationModel{
{
Model: &logicalRouter,
ModelPredicate: func(lr *nbdb.LogicalRouter) bool { return lr.Name == gatewayRouter },
OnModelUpdates: []interface{}{
&logicalRouter.Options,
&logicalRouter.ExternalIDs,
&logicalRouter.LoadBalancerGroup,
},
},
}
Expand Down
36 changes: 33 additions & 3 deletions go-controller/pkg/ovn/gateway_test.go
Expand Up @@ -160,9 +160,10 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus
"physical_ip": physicalIPs[0],
"physical_ips": strings.Join(physicalIPs, ","),
},
Ports: []string{gwRouterPort + "-UUID", externalRouterPort + "-UUID"},
StaticRoutes: grStaticRoutes,
Nat: natUUIDs,
Ports: []string{gwRouterPort + "-UUID", externalRouterPort + "-UUID"},
StaticRoutes: grStaticRoutes,
Nat: natUUIDs,
LoadBalancerGroup: []string{types.ClusterLBGroupName + "-UUID"},
})

testData = append(testData, expectedOVNClusterRouter)
Expand Down Expand Up @@ -218,6 +219,10 @@ func generateGatewayInitExpectedNB(testData []libovsdb.TestData, expectedOVNClus
UUID: externalSwitch + "-UUID",
Name: externalSwitch,
Ports: []string{l3GatewayConfig.InterfaceID + "-UUID", externalSwitchPortToRouter + "-UUID"},
},
&nbdb.LoadBalancerGroup{
Name: types.ClusterLBGroupName,
UUID: types.ClusterLBGroupName + "-UUID",
})
return testData
}
Expand Down Expand Up @@ -254,6 +259,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
UUID: nodeName + "-UUID",
Name: nodeName,
}
expectedClusterLBGroup := &nbdb.LoadBalancerGroup{
UUID: types.ClusterLBGroupName + "-UUID",
Name: types.ClusterLBGroupName,
}
fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{
NBData: []libovsdbtest.TestData{
&nbdb.LogicalSwitch{
Expand All @@ -262,6 +271,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
},
expectedOVNClusterRouter,
expectedNodeSwitch,
expectedClusterLBGroup,
},
})

Expand Down Expand Up @@ -300,6 +310,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
UUID: nodeName + "-UUID",
Name: nodeName,
}
expectedClusterLBGroup := &nbdb.LoadBalancerGroup{
UUID: types.ClusterLBGroupName + "-UUID",
Name: types.ClusterLBGroupName,
}
fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{
NBData: []libovsdbtest.TestData{
&nbdb.LogicalSwitch{
Expand All @@ -308,6 +322,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
},
expectedOVNClusterRouter,
expectedNodeSwitch,
expectedClusterLBGroup,
},
})

Expand Down Expand Up @@ -347,6 +362,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
UUID: nodeName + "-UUID",
Name: nodeName,
}
expectedClusterLBGroup := &nbdb.LoadBalancerGroup{
UUID: types.ClusterLBGroupName + "-UUID",
Name: types.ClusterLBGroupName,
}
fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{
NBData: []libovsdbtest.TestData{
&nbdb.LogicalSwitch{
Expand All @@ -355,6 +374,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
},
expectedOVNClusterRouter,
expectedNodeSwitch,
expectedClusterLBGroup,
},
})

Expand Down Expand Up @@ -394,6 +414,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
UUID: nodeName + "-UUID",
Name: nodeName,
}
expectedClusterLBGroup := &nbdb.LoadBalancerGroup{
UUID: types.ClusterLBGroupName + "-UUID",
Name: types.ClusterLBGroupName,
}
fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{
NBData: []libovsdbtest.TestData{
&nbdb.LogicalSwitch{
Expand All @@ -402,6 +426,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
},
expectedOVNClusterRouter,
expectedNodeSwitch,
expectedClusterLBGroup,
},
})

Expand Down Expand Up @@ -450,6 +475,10 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
UUID: nodeName + "-UUID",
Name: nodeName,
}
expectedClusterLBGroup := &nbdb.LoadBalancerGroup{
Name: types.ClusterLBGroupName,
UUID: types.ClusterLBGroupName + "-UUID",
}
fakeOvn.startWithDBSetup(libovsdbtest.TestSetup{
NBData: []libovsdbtest.TestData{
&nbdb.LogicalSwitch{
Expand All @@ -458,6 +487,7 @@ var _ = ginkgo.Describe("Gateway Init Operations", func() {
},
expectedOVNClusterRouter,
expectedNodeSwitch,
expectedClusterLBGroup,
},
})

Expand Down
115 changes: 115 additions & 0 deletions go-controller/pkg/ovn/libovsdbops/lbgroup.go
@@ -0,0 +1,115 @@
package libovsdbops

import (
"context"
"fmt"

libovsdbclient "github.com/ovn-org/libovsdb/client"
"github.com/ovn-org/libovsdb/model"
libovsdb "github.com/ovn-org/libovsdb/ovsdb"

"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/nbdb"
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
)

// findLBGroup looks up the Group in the cache and sets the UUID
func findLBGroup(nbClient libovsdbclient.Client, group *nbdb.LoadBalancerGroup) error {
if group.UUID != "" && !IsNamedUUID(group.UUID) {
return nil
}

ctx, cancel := context.WithTimeout(context.Background(), types.OVSDBTimeout)
defer cancel()
groups := []nbdb.LoadBalancerGroup{}
err := nbClient.WhereCache(func(item *nbdb.LoadBalancerGroup) bool {
return item.Name == group.Name
}).List(ctx, &groups)
if err != nil {
return fmt.Errorf("can't find LB group %+v: %v", *group, err)
}

if len(groups) > 1 {
return fmt.Errorf("unexpectedly found multiple LB Groups: %+v", groups)
}

if len(groups) == 0 {
return libovsdbclient.ErrNotFound
}

group.UUID = groups[0].UUID
return nil
}

func AddLoadBalancersToGroupOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, group *nbdb.LoadBalancerGroup, lbs ...*nbdb.LoadBalancer) ([]libovsdb.Operation, error) {
if ops == nil {
ops = []libovsdb.Operation{}
}
if len(lbs) == 0 {
return ops, nil
}

err := findLBGroup(nbClient, group)
if err != nil {
return nil, err
}

lbUUIDs := make([]string, 0, len(lbs))
for _, lb := range lbs {
lbUUIDs = append(lbUUIDs, lb.UUID)
}

op, err := nbClient.Where(group).Mutate(group, model.Mutation{
Field: &group.LoadBalancer,
Mutator: libovsdb.MutateOperationInsert,
Value: lbUUIDs,
})
if err != nil {
return nil, err
}
ops = append(ops, op...)
return ops, nil
}

func RemoveLoadBalancersFromGroupOps(nbClient libovsdbclient.Client, ops []libovsdb.Operation, group *nbdb.LoadBalancerGroup, lbs ...*nbdb.LoadBalancer) ([]libovsdb.Operation, error) {
if ops == nil {
ops = []libovsdb.Operation{}
}
if len(lbs) == 0 {
return ops, nil
}

err := findLBGroup(nbClient, group)
if err != nil {
return nil, err
}

lbUUIDs := make([]string, 0, len(lbs))
for _, lb := range lbs {
lbUUIDs = append(lbUUIDs, lb.UUID)
}

op, err := nbClient.Where(group).Mutate(group, model.Mutation{
Field: &group.LoadBalancer,
Mutator: libovsdb.MutateOperationDelete,
Value: lbUUIDs,
})
if err != nil {
return nil, err
}
ops = append(ops, op...)

return ops, nil
}

func ListGroupsWithLoadBalancers(nbClient libovsdbclient.Client) ([]nbdb.LoadBalancerGroup, error) {
groups := &[]nbdb.LoadBalancerGroup{}
ctx, cancel := context.WithTimeout(context.Background(), types.OVSDBTimeout)
defer cancel()
err := nbClient.WhereCache(func(item *nbdb.LoadBalancerGroup) bool {
return item.LoadBalancer != nil
}).List(ctx, groups)
if err != nil {
return nil, err
}
return *groups, nil
}

0 comments on commit 18d11d4

Please sign in to comment.