From 2859fd87178860df4b0f621182faa109c853f84a Mon Sep 17 00:00:00 2001 From: "O. Yuanying" Date: Wed, 11 Aug 2021 10:27:44 +0000 Subject: [PATCH] host-gw (linux): Add dual stack support Add dual stack support to host-gw backend. Currently only work with linux now. Other limitations are the same as for the vxlan backend. Signed-off-by: O. Yuanying --- Documentation/configuration.md | 6 +- backend/hostgw/hostgw.go | 30 +++++-- backend/route_network.go | 150 +++++++++++++++++++++++---------- backend/route_network_test.go | 59 ++++++++++++- subnet/kube/kube.go | 19 +++-- 5 files changed, 199 insertions(+), 65 deletions(-) diff --git a/Documentation/configuration.md b/Documentation/configuration.md index 984e268ab..e97fc3e24 100644 --- a/Documentation/configuration.md +++ b/Documentation/configuration.md @@ -83,7 +83,7 @@ Set `healthz-port` to a non-zero value will enable a healthz server for flannel. ## Dual-stack -Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan backend. +Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan or host-gw(linux) backend. Requirements: * v1.0 of flannel binary from [containernetworking/plugins](https://github.com/containernetworking/plugins) @@ -93,13 +93,15 @@ Requirements: Configuration: * Set flanneld daemon with "--kube-subnet-mgr" CLI option -* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap +* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap If everything works as expected, flanneld should generate a `/run/flannel/subnet.env` file with IPV6 subnet and network. For example: +```bash FLANNEL_NETWORK=10.42.0.0/16 FLANNEL_SUBNET=10.42.0.1/24 FLANNEL_IPV6_NETWORK=2001:cafe:42::/56 FLANNEL_IPV6_SUBNET=2001:cafe:42::1/64 FLANNEL_MTU=1450 FLANNEL_IPMASQ=true +``` diff --git a/backend/hostgw/hostgw.go b/backend/hostgw/hostgw.go index d10639d45..5a528abf2 100644 --- a/backend/hostgw/hostgw.go +++ b/backend/hostgw/hostgw.go @@ -59,19 +59,33 @@ func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup Mtu: be.extIface.Iface.MTU, LinkIndex: be.extIface.Iface.Index, } - n.GetRoute = func(lease *subnet.Lease) *netlink.Route { - return &netlink.Route{ - Dst: lease.Subnet.ToIPNet(), - Gw: lease.Attrs.PublicIP.ToIP(), - LinkIndex: n.LinkIndex, - } - } attrs := subnet.LeaseAttrs{ - PublicIP: ip.FromIP(be.extIface.ExtAddr), BackendType: "host-gw", } + if config.EnableIPv4 { + attrs.PublicIP = ip.FromIP(be.extIface.ExtAddr) + n.GetRoute = func(lease *subnet.Lease) *netlink.Route { + return &netlink.Route{ + Dst: lease.Subnet.ToIPNet(), + Gw: lease.Attrs.PublicIP.ToIP(), + LinkIndex: n.LinkIndex, + } + } + } + + if config.EnableIPv6 { + attrs.PublicIPv6 = ip.FromIP6(be.extIface.ExtV6Addr) + n.GetV6Route = func(lease *subnet.Lease) *netlink.Route { + return &netlink.Route{ + Dst: lease.IPv6Subnet.ToIPNet(), + Gw: lease.Attrs.PublicIPv6.ToIP(), + LinkIndex: n.LinkIndex, + } + } + } + l, err := be.sm.AcquireLease(ctx, &attrs) switch err { case nil: diff --git a/backend/route_network.go b/backend/route_network.go index fb60398e2..26dfa90a3 100644 --- a/backend/route_network.go +++ b/backend/route_network.go @@ -36,8 +36,10 @@ type RouteNetwork struct { SimpleNetwork BackendType string routes []netlink.Route + v6Routes []netlink.Route SM subnet.Manager GetRoute func(lease *subnet.Lease) *netlink.Route + GetV6Route func(lease *subnet.Lease) *netlink.Route Mtu int LinkIndex int } @@ -82,54 +84,53 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) { for _, evt := range batch { switch evt.Type { case subnet.EventAdded: - log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP) - if evt.Lease.Attrs.BackendType != n.BackendType { log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType) continue } - route := n.GetRoute(&evt.Lease) - n.addToRouteList(*route) - // Check if route exists before attempting to add it - routeList, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST) - if err != nil { - log.Warningf("Unable to list routes: %v", err) - } + if evt.Lease.EnableIPv4 { + log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP) - if len(routeList) > 0 && !routeEqual(routeList[0], *route) { - // Same Dst different Gw or different link index. Remove it, correct route will be added below. - log.Warningf("Replacing existing route to %v via %v dev index %d with %v via %v dev index %d.", evt.Lease.Subnet, routeList[0].Gw, routeList[0].LinkIndex, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex) - if err := netlink.RouteDel(&routeList[0]); err != nil { - log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err) - continue - } - n.removeFromRouteList(routeList[0]) + route := n.GetRoute(&evt.Lease) + routeAdd(route, netlink.FAMILY_V4, n.addToRouteList, n.removeFromV4RouteList) } - if len(routeList) > 0 && routeEqual(routeList[0], *route) { - // Same Dst and same Gw, keep it and do not attempt to add it. - log.Infof("Route to %v via %v dev index %d already exists, skipping.", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, routeList[0].LinkIndex) - } else if err := netlink.RouteAdd(route); err != nil { - log.Errorf("Error adding route to %v via %v dev index %d: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex, err) - continue + if evt.Lease.EnableIPv6 { + log.Infof("Subnet added: %v via %v", evt.Lease.IPv6Subnet, evt.Lease.Attrs.PublicIPv6) + + route := n.GetV6Route(&evt.Lease) + routeAdd(route, netlink.FAMILY_V6, n.addToV6RouteList, n.removeFromV6RouteList) } case subnet.EventRemoved: - log.Info("Subnet removed: ", evt.Lease.Subnet) - if evt.Lease.Attrs.BackendType != n.BackendType { log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType) continue } - route := n.GetRoute(&evt.Lease) - // Always remove the route from the route list. - n.removeFromRouteList(*route) + if evt.Lease.EnableIPv4 { + log.Info("Subnet removed: ", evt.Lease.Subnet) - if err := netlink.RouteDel(route); err != nil { - log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err) - continue + route := n.GetRoute(&evt.Lease) + // Always remove the route from the route list. + n.removeFromV4RouteList(*route) + + if err := netlink.RouteDel(route); err != nil { + log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err) + } + } + + if evt.Lease.EnableIPv6 { + log.Info("Subnet removed: ", evt.Lease.IPv6Subnet) + + route := n.GetV6Route(&evt.Lease) + // Always remove the route from the route list. + n.removeFromV6RouteList(*route) + + if err := netlink.RouteDel(route); err != nil { + log.Errorf("Error deleting route to %v: %v", evt.Lease.IPv6Subnet, err) + } } default: @@ -138,22 +139,74 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) { } } -func (n *RouteNetwork) addToRouteList(route netlink.Route) { - for _, r := range n.routes { - if routeEqual(r, route) { +func routeAdd(route *netlink.Route, ipFamily int, addToRouteList, removeFromRouteList func(netlink.Route)) { + addToRouteList(*route) + // Check if route exists before attempting to add it + routeList, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST) + if err != nil { + log.Warningf("Unable to list routes: %v", err) + } + + if len(routeList) > 0 && !routeEqual(routeList[0], *route) { + // Same Dst different Gw or different link index. Remove it, correct route will be added below. + log.Warningf("Replacing existing route to %v with %v", routeList[0], route) + if err := netlink.RouteDel(&routeList[0]); err != nil { + log.Errorf("Effor deleteing route to %v: %v", routeList[0].Dst, err) return } + removeFromRouteList(routeList[0]) + } + routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST) + if err != nil { + log.Warningf("Unable to list routes: %v", err) + } + + if len(routeList) > 0 && routeEqual(routeList[0], *route) { + // Same Dst and same Gw, keep it and do not attempt to add it. + log.Infof("Route to %v already exists, skipping.", route) + } else if err := netlink.RouteAdd(route); err != nil { + log.Errorf("Error adding route to %v", route) + return + } + routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST) + if err != nil { + log.Warningf("Unable to list routes: %v", err) + } +} + +func (n *RouteNetwork) addToRouteList(route netlink.Route) { + n.routes = addToRouteList(&route, n.routes) +} + +func (n *RouteNetwork) addToV6RouteList(route netlink.Route) { + n.v6Routes = addToRouteList(&route, n.v6Routes) +} + +func addToRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route { + for _, r := range routes { + if routeEqual(r, *route) { + return routes + } } - n.routes = append(n.routes, route) + return append(routes, *route) } -func (n *RouteNetwork) removeFromRouteList(route netlink.Route) { - for index, r := range n.routes { - if routeEqual(r, route) { - n.routes = append(n.routes[:index], n.routes[index+1:]...) - return +func (n *RouteNetwork) removeFromV4RouteList(route netlink.Route) { + n.routes = n.removeFromRouteList(&route, n.routes) +} + +func (n *RouteNetwork) removeFromV6RouteList(route netlink.Route) { + n.v6Routes = n.removeFromRouteList(&route, n.v6Routes) +} + +func (n *RouteNetwork) removeFromRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route { + for index, r := range routes { + if routeEqual(r, *route) { + routes = append(routes[:index], routes[index+1:]...) + return routes } } + return routes } func (n *RouteNetwork) routeCheck(ctx context.Context) { @@ -162,15 +215,24 @@ func (n *RouteNetwork) routeCheck(ctx context.Context) { case <-ctx.Done(): return case <-time.After(routeCheckRetries * time.Second): - n.checkSubnetExistInRoutes() + n.checkSubnetExistInV4Routes() + n.checkSubnetExistInV6Routes() } } } -func (n *RouteNetwork) checkSubnetExistInRoutes() { - routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4) +func (n *RouteNetwork) checkSubnetExistInV4Routes() { + n.checkSubnetExistInRoutes(n.routes, netlink.FAMILY_V4) +} + +func (n *RouteNetwork) checkSubnetExistInV6Routes() { + n.checkSubnetExistInRoutes(n.v6Routes, netlink.FAMILY_V6) +} + +func (n *RouteNetwork) checkSubnetExistInRoutes(routes []netlink.Route, ipFamily int) { + routeList, err := netlink.RouteList(nil, ipFamily) if err == nil { - for _, route := range n.routes { + for _, route := range routes { exist := false for _, r := range routeList { if r.Dst == nil { diff --git a/backend/route_network_test.go b/backend/route_network_test.go index 41f52805e..6301939ae 100644 --- a/backend/route_network_test.go +++ b/backend/route_network_test.go @@ -57,7 +57,7 @@ func TestRouteCache(t *testing.T) { subnet1 := ip.IP4Net{IP: ip.FromIP(net.ParseIP("192.168.0.0")), PrefixLen: 24} nw.handleSubnetEvents([]subnet.Event{ {Type: subnet.EventAdded, Lease: subnet.Lease{ - Subnet: subnet1, Attrs: subnet.LeaseAttrs{PublicIP: gw1, BackendType: "host-gw"}}}, + Subnet: subnet1, EnableIPv4: true, Attrs: subnet.LeaseAttrs{PublicIP: gw1, BackendType: "host-gw"}}}, }) if len(nw.routes) != 1 { t.Fatal(nw.routes) @@ -68,7 +68,7 @@ func TestRouteCache(t *testing.T) { // change gateway of previous route nw.handleSubnetEvents([]subnet.Event{ {Type: subnet.EventAdded, Lease: subnet.Lease{ - Subnet: subnet1, Attrs: subnet.LeaseAttrs{PublicIP: gw2, BackendType: "host-gw"}}}}) + Subnet: subnet1, EnableIPv4: true, Attrs: subnet.LeaseAttrs{PublicIP: gw2, BackendType: "host-gw"}}}}) if len(nw.routes) != 1 { t.Fatal(nw.routes) } @@ -76,3 +76,58 @@ func TestRouteCache(t *testing.T) { t.Fatal(nw.routes[0]) } } + +func TestV6RouteCache(t *testing.T) { + teardown := ns.SetUpNetlinkTest(t) + defer teardown() + + la := netlink.NewLinkAttrs() + la.Name = "br" + br := &netlink.Bridge{LinkAttrs: la} + if err := netlink.LinkAdd(br); err != nil { + t.Fatal(err) + } + if err := netlink.AddrAdd(br, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::1"), Mask: net.CIDRMask(64, 128)}}); err != nil { + t.Fatal(err) + } + if err := netlink.LinkSetUp(br); err != nil { + t.Fatal(err) + } + + nw := RouteNetwork{ + SimpleNetwork: SimpleNetwork{ + ExtIface: &ExternalInterface{Iface: &net.Interface{Index: br.Attrs().Index}}, + }, + BackendType: "host-gw", + LinkIndex: br.Attrs().Index, + } + nw.GetV6Route = func(lease *subnet.Lease) *netlink.Route { + return &netlink.Route{ + Dst: lease.IPv6Subnet.ToIPNet(), + Gw: lease.Attrs.PublicIPv6.ToIP(), + LinkIndex: nw.LinkIndex, + } + } + gw1, gw2 := ip.FromIP6(net.ParseIP("2001:db8:1::2")), ip.FromIP6(net.ParseIP("::2")) + subnet1 := ip.IP6Net{IP: ip.FromIP6(net.ParseIP("2001:db8:ffff::")), PrefixLen: 64} + nw.handleSubnetEvents([]subnet.Event{ + {Type: subnet.EventAdded, Lease: subnet.Lease{ + IPv6Subnet: subnet1, EnableIPv6: true, Attrs: subnet.LeaseAttrs{PublicIPv6: gw1, BackendType: "host-gw"}}}, + }) + if len(nw.v6Routes) != 1 { + t.Fatal(nw.v6Routes) + } + if !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw1.ToIP(), LinkIndex: br.Attrs().Index}) { + t.Fatal(nw.v6Routes[0]) + } + // change gateway of previous route + nw.handleSubnetEvents([]subnet.Event{ + {Type: subnet.EventAdded, Lease: subnet.Lease{ + IPv6Subnet: subnet1, EnableIPv6: true, Attrs: subnet.LeaseAttrs{PublicIPv6: gw2, BackendType: "host-gw"}}}}) + if len(nw.v6Routes) != 1 { + t.Fatal(nw.v6Routes) + } + if !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw2.ToIP(), LinkIndex: br.Attrs().Index}) { + t.Fatal(nw.v6Routes[0]) + } +} diff --git a/subnet/kube/kube.go b/subnet/kube/kube.go index 1118b486b..eed5c6534 100644 --- a/subnet/kube/kube.go +++ b/subnet/kube/kube.go @@ -277,14 +277,15 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() || n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" || (n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String())) || - (n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) || - n.Annotations[ksm.annotations.BackendType] != attrs.BackendType || - n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() || - n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" || - (n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String())) { + (attrs.PublicIPv6 != nil && + (n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) || + n.Annotations[ksm.annotations.BackendType] != attrs.BackendType || + n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() || + n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" || + (n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String()))) { n.Annotations[ksm.annotations.BackendType] = attrs.BackendType - //TODO -i only vxlan backend support dual stack now. + //TODO -i only vxlan and host-gw backends support dual stack now. if (attrs.BackendType == "vxlan" && string(bd) != "null") || attrs.BackendType != "vxlan" { n.Annotations[ksm.annotations.BackendData] = string(bd) if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" { @@ -299,7 +300,7 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le } } - if string(v6Bd) != "null" { + if (attrs.BackendType == "vxlan" && string(v6Bd) != "null") || (attrs.BackendType == "host-gw" && attrs.PublicIPv6 != nil) { n.Annotations[ksm.annotations.BackendV6Data] = string(v6Bd) if n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" { if n.Annotations[ksm.annotations.BackendPublicIPv6] != n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] { @@ -354,8 +355,8 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le if ipv6Cidr != nil { lease.IPv6Subnet = ip.FromIP6Net(ipv6Cidr) } - //TODO - only vxlan backend support dual stack now. - if attrs.BackendType != "vxlan" { + //TODO - only vxlan and host-gw backends support dual stack now. + if attrs.BackendType != "vxlan" && attrs.BackendType != "host-gw" { lease.EnableIPv4 = true lease.EnableIPv6 = false }