From 67254ad22d3a55fbd051cb01228e9515ebf5891e Mon Sep 17 00:00:00 2001 From: Aaron U'Ren Date: Mon, 2 Jan 2023 15:42:01 -0600 Subject: [PATCH] fix(ecmp_vip): handle ipv4 & ipv6 protocols --- go.mod | 8 ++--- go.sum | 15 ++------ pkg/controllers/routing/ecmp_vip.go | 54 +++++++++++++++++++++-------- pkg/controllers/routing/utils.go | 35 +++++++++++++++++++ 4 files changed, 80 insertions(+), 32 deletions(-) diff --git a/go.mod b/go.mod index 5271a4072..1a5115f29 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/cloudnativelabs/kube-router require ( github.com/aws/aws-sdk-go v1.45.1 - github.com/containernetworking/cni v1.1.2 - github.com/containernetworking/plugins v1.3.0 github.com/coreos/go-iptables v0.7.0 github.com/docker/docker v24.0.5+incompatible github.com/moby/ipvs v1.1.0 @@ -15,6 +13,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/vishvananda/netlink v1.2.1-beta.2 github.com/vishvananda/netns v0.0.4 + golang.org/x/exp v0.0.0-20230307190834-24139beb5833 golang.org/x/net v0.14.0 google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 @@ -23,7 +22,7 @@ require ( k8s.io/client-go v0.27.5 k8s.io/cri-api v0.27.5 k8s.io/klog/v2 v2.100.1 - k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 + k8s.io/utils v0.0.0-20230209194617-a36077c30491 ) require ( @@ -49,6 +48,7 @@ require ( github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect + github.com/google/pprof v0.0.0-20230323073829-e72429f035bd // indirect github.com/google/uuid v1.3.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -74,7 +74,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect - github.com/safchain/ethtool v0.3.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect github.com/spf13/afero v1.9.5 // indirect github.com/spf13/cast v1.5.0 // indirect @@ -97,7 +96,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools/v3 v3.4.0 // indirect k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect - k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index a01e4a1ec..88574e2e5 100644 --- a/go.sum +++ b/go.sum @@ -55,10 +55,6 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= -github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM= -github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0= github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8= github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -173,8 +169,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= @@ -240,14 +236,11 @@ github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= -github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -274,8 +267,6 @@ github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+Pymzi github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= -github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= @@ -338,6 +329,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230307190834-24139beb5833 h1:SChBja7BCQewoTAU7IgvucQKMIXrEpFxNMs0spT3/5s= +golang.org/x/exp v0.0.0-20230307190834-24139beb5833/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -397,7 +390,6 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= @@ -473,7 +465,6 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pkg/controllers/routing/ecmp_vip.go b/pkg/controllers/routing/ecmp_vip.go index efff930c9..9168a20fc 100644 --- a/pkg/controllers/routing/ecmp_vip.go +++ b/pkg/controllers/routing/ecmp_vip.go @@ -6,6 +6,8 @@ import ( "fmt" "strconv" + "golang.org/x/exp/maps" + "google.golang.org/protobuf/types/known/anypb" "github.com/cloudnativelabs/kube-router/pkg/metrics" @@ -22,24 +24,27 @@ import ( // bgpAdvertiseVIP advertises the service vip (cluster ip or load balancer ip or external IP) the configured peers func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error { + subnet, nh, afiFamily, err := nrc.getBGPRouteInfoForVIP(vip) + if err != nil { + return fmt.Errorf("unable to advertise VIP because of: %v", err) + } - klog.V(2).Infof("Advertising route: '%s/%s via %s' to peers", - vip, strconv.Itoa(32), nrc.primaryIP.String()) + klog.V(2).Infof("Advertising route: '%s/%d via %s' to peers", vip, subnet, nh) a1, _ := anypb.New(&gobgpapi.OriginAttribute{ Origin: 0, }) a2, _ := anypb.New(&gobgpapi.NextHopAttribute{ - NextHop: nrc.primaryIP.String(), + NextHop: nh, }) attrs := []*anypb.Any{a1, a2} nlri1, _ := anypb.New(&gobgpapi.IPAddressPrefix{ Prefix: vip, - PrefixLen: 32, + PrefixLen: subnet, }) - _, err := nrc.bgpServer.AddPath(context.Background(), &gobgpapi.AddPathRequest{ + _, err = nrc.bgpServer.AddPath(context.Background(), &gobgpapi.AddPathRequest{ Path: &gobgpapi.Path{ - Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST}, + Family: &gobgpapi.Family{Afi: afiFamily, Safi: gobgpapi.Family_SAFI_UNICAST}, Nlri: nlri1, Pattrs: attrs, }, @@ -52,28 +57,32 @@ func (nrc *NetworkRoutingController) bgpAdvertiseVIP(vip string) error { return err } -// bgpWithdrawVIP unadvertises the service vip +// bgpWithdrawVIP un-advertises the service vip func (nrc *NetworkRoutingController) bgpWithdrawVIP(vip string) error { - klog.V(2).Infof("Withdrawing route: '%s/%s via %s' to peers", - vip, strconv.Itoa(32), nrc.primaryIP.String()) + subnet, nh, afiFamily, err := nrc.getBGPRouteInfoForVIP(vip) + if err != nil { + return fmt.Errorf("unable to advertise VIP because of: %v", err) + } + + klog.V(2).Infof("Withdrawing route: '%s/%d via %s' to peers", vip, subnet, nh) a1, _ := anypb.New(&gobgpapi.OriginAttribute{ Origin: 0, }) a2, _ := anypb.New(&gobgpapi.NextHopAttribute{ - NextHop: nrc.primaryIP.String(), + NextHop: nh, }) attrs := []*anypb.Any{a1, a2} nlri, _ := anypb.New(&gobgpapi.IPAddressPrefix{ Prefix: vip, - PrefixLen: 32, + PrefixLen: subnet, }) path := gobgpapi.Path{ - Family: &gobgpapi.Family{Afi: gobgpapi.Family_AFI_IP, Safi: gobgpapi.Family_SAFI_UNICAST}, + Family: &gobgpapi.Family{Afi: afiFamily, Safi: gobgpapi.Family_SAFI_UNICAST}, Nlri: nlri, Pattrs: attrs, } - err := nrc.bgpServer.DeletePath(context.Background(), &gobgpapi.DeletePathRequest{ + err = nrc.bgpServer.DeletePath(context.Background(), &gobgpapi.DeletePathRequest{ TableType: gobgpapi.TableType_GLOBAL, Path: &path, }) @@ -541,6 +550,19 @@ func (nrc *NetworkRoutingController) nodeHasEndpointsForService(svc *v1core.Serv return false, errors.New("failed to convert cache item to Endpoints type") } + // Find all the IPs that this node has on it so that we can use it to compare it against endpoint IPs + allNodeIPs := make([]string, 0) + for _, ips := range maps.Values(nrc.nodeIPv4Addrs) { + for _, ip := range ips { + allNodeIPs = append(allNodeIPs, ip.String()) + } + } + for _, ips := range maps.Values(nrc.nodeIPv6Addrs) { + for _, ip := range ips { + allNodeIPs = append(allNodeIPs, ip.String()) + } + } + for _, subset := range ep.Subsets { for _, address := range subset.Addresses { if address.NodeName != nil { @@ -548,8 +570,10 @@ func (nrc *NetworkRoutingController) nodeHasEndpointsForService(svc *v1core.Serv return true, nil } } else { - if address.IP == nrc.primaryIP.String() { - return true, nil + for _, nodeIP := range allNodeIPs { + if address.IP == nodeIP { + return true, nil + } } } } diff --git a/pkg/controllers/routing/utils.go b/pkg/controllers/routing/utils.go index 338090007..e99cc4259 100644 --- a/pkg/controllers/routing/utils.go +++ b/pkg/controllers/routing/utils.go @@ -10,6 +10,7 @@ import ( "strconv" "strings" + "github.com/cloudnativelabs/kube-router/pkg/utils" gobgpapi "github.com/osrg/gobgp/v3/api" "github.com/osrg/gobgp/v3/pkg/packet/bgp" "github.com/vishvananda/netlink/nl" @@ -258,3 +259,37 @@ func getPodCIDRsFromAllNodeSources(node *v1core.Node) (podCIDRs []string) { // Finally, if all else fails, use the PodCIDRs on the node spec return node.Spec.PodCIDRs } + +// getBGPRouteInfoForVIP attempt to automatically find the subnet, BGP AFI/SAFI Family, and nexthop for a given VIP +// based upon whether it is an IPv4 address or an IPv6 address. Returns slash notation subnet as uint32 suitable for +// sending to GoBGP and an error if it is unable to determine the subnet automatically +func (nrc *NetworkRoutingController) getBGPRouteInfoForVIP(vip string) (subnet uint32, nh string, + afiFamily gobgpapi.Family_Afi, err error) { + ip := net.ParseIP(vip) + if ip == nil { + err = fmt.Errorf("could not parse VIP: %s", vip) + return + } + if ip.To4() != nil { + subnet = 32 + afiFamily = gobgpapi.Family_AFI_IP + nhIP := utils.FindBestIPv4NodeAddress(nrc.primaryIP, nrc.nodeIPv4Addrs) + if nhIP == nil { + err = fmt.Errorf("could not find an IPv4 address on node to set as nexthop for vip: %s", vip) + } + nh = nhIP.String() + return + } + if ip.To16() != nil { + subnet = 128 + afiFamily = gobgpapi.Family_AFI_IP6 + nhIP := utils.FindBestIPv6NodeAddress(nrc.primaryIP, nrc.nodeIPv6Addrs) + if nhIP == nil { + err = fmt.Errorf("could not find an IPv6 address on node to set as nexthop for vip: %s", vip) + } + nh = nhIP.String() + return + } + err = fmt.Errorf("could not convert IP to IPv4 or IPv6, unable to find subnet for: %s", vip) + return +}