Skip to content

Commit

Permalink
host-gw (linux): Add dual stack support
Browse files Browse the repository at this point in the history 
Add dual stack support to host-gw backend. Currently only work with
linux now. Other limitations are the same as for the vxlan backend.

Signed-off-by: O. Yuanying <yuanying@fraction.jp>
  • Loading branch information
@yuanying
yuanying committed Aug 17, 2021
1 parent b75dc70 commit 2859fd8
Show file tree
Hide file tree
Showing 5 changed files with 199 additions and 65 deletions.
6 changes: 4 additions & 2 deletions Documentation/configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ Set `healthz-port` to a non-zero value will enable a healthz server for flannel.
## Dual-stack
Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan backend.
Flannel supports the dual-stack mode of Kubernetes. This means pods and services could use ipv4 and ipv6 at the same time. Currently, dual-stack is only supported for kube subnet manager and vxlan or host-gw(linux) backend.
Requirements:
* v1.0 of flannel binary from [containernetworking/plugins](https://github.com/containernetworking/plugins)
Expand All @@ -93,13 +93,15 @@ Requirements:
Configuration:
* Set flanneld daemon with "--kube-subnet-mgr" CLI option
* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap
* Set "EnableIPv6": true and the "IPv6Network", for example "IPv6Network": "2001:cafe:42:0::/56" in the net-conf.json of the kube-flannel-cfg ConfigMap
If everything works as expected, flanneld should generate a `/run/flannel/subnet.env` file with IPV6 subnet and network. For example:
```bash
FLANNEL_NETWORK=10.42.0.0/16
FLANNEL_SUBNET=10.42.0.1/24
FLANNEL_IPV6_NETWORK=2001:cafe:42::/56
FLANNEL_IPV6_SUBNET=2001:cafe:42::1/64
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
```
30 changes: 22 additions & 8 deletions backend/hostgw/hostgw.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,19 +59,33 @@ func (be *HostgwBackend) RegisterNetwork(ctx context.Context, wg *sync.WaitGroup
Mtu: be.extIface.Iface.MTU,
LinkIndex: be.extIface.Iface.Index,
}
n.GetRoute = func(lease *subnet.Lease) *netlink.Route {
return &netlink.Route{
Dst: lease.Subnet.ToIPNet(),
Gw: lease.Attrs.PublicIP.ToIP(),
LinkIndex: n.LinkIndex,
}
}

attrs := subnet.LeaseAttrs{
PublicIP: ip.FromIP(be.extIface.ExtAddr),
BackendType: "host-gw",
}

if config.EnableIPv4 {
attrs.PublicIP = ip.FromIP(be.extIface.ExtAddr)
n.GetRoute = func(lease *subnet.Lease) *netlink.Route {
return &netlink.Route{
Dst: lease.Subnet.ToIPNet(),
Gw: lease.Attrs.PublicIP.ToIP(),
LinkIndex: n.LinkIndex,
}
}
}

if config.EnableIPv6 {
attrs.PublicIPv6 = ip.FromIP6(be.extIface.ExtV6Addr)
n.GetV6Route = func(lease *subnet.Lease) *netlink.Route {
return &netlink.Route{
Dst: lease.IPv6Subnet.ToIPNet(),
Gw: lease.Attrs.PublicIPv6.ToIP(),
LinkIndex: n.LinkIndex,
}
}
}

l, err := be.sm.AcquireLease(ctx, &attrs)
switch err {
case nil:
Expand Down
150 changes: 106 additions & 44 deletions backend/route_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,10 @@ type RouteNetwork struct {
SimpleNetwork
BackendType string
routes []netlink.Route
v6Routes []netlink.Route
SM subnet.Manager
GetRoute func(lease *subnet.Lease) *netlink.Route
GetV6Route func(lease *subnet.Lease) *netlink.Route
Mtu int
LinkIndex int
}
Expand Down Expand Up @@ -82,54 +84,53 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) {
for _, evt := range batch {
switch evt.Type {
case subnet.EventAdded:
log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)

if evt.Lease.Attrs.BackendType != n.BackendType {
log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType)
continue
}
route := n.GetRoute(&evt.Lease)

n.addToRouteList(*route)
// Check if route exists before attempting to add it
routeList, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
if err != nil {
log.Warningf("Unable to list routes: %v", err)
}
if evt.Lease.EnableIPv4 {
log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)

if len(routeList) > 0 && !routeEqual(routeList[0], *route) {
// Same Dst different Gw or different link index. Remove it, correct route will be added below.
log.Warningf("Replacing existing route to %v via %v dev index %d with %v via %v dev index %d.", evt.Lease.Subnet, routeList[0].Gw, routeList[0].LinkIndex, evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex)
if err := netlink.RouteDel(&routeList[0]); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
continue
}
n.removeFromRouteList(routeList[0])
route := n.GetRoute(&evt.Lease)
routeAdd(route, netlink.FAMILY_V4, n.addToRouteList, n.removeFromV4RouteList)
}

if len(routeList) > 0 && routeEqual(routeList[0], *route) {
// Same Dst and same Gw, keep it and do not attempt to add it.
log.Infof("Route to %v via %v dev index %d already exists, skipping.", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, routeList[0].LinkIndex)
} else if err := netlink.RouteAdd(route); err != nil {
log.Errorf("Error adding route to %v via %v dev index %d: %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP, route.LinkIndex, err)
continue
if evt.Lease.EnableIPv6 {
log.Infof("Subnet added: %v via %v", evt.Lease.IPv6Subnet, evt.Lease.Attrs.PublicIPv6)

route := n.GetV6Route(&evt.Lease)
routeAdd(route, netlink.FAMILY_V6, n.addToV6RouteList, n.removeFromV6RouteList)
}

case subnet.EventRemoved:
log.Info("Subnet removed: ", evt.Lease.Subnet)

if evt.Lease.Attrs.BackendType != n.BackendType {
log.Warningf("Ignoring non-%v subnet: type=%v", n.BackendType, evt.Lease.Attrs.BackendType)
continue
}

route := n.GetRoute(&evt.Lease)
// Always remove the route from the route list.
n.removeFromRouteList(*route)
if evt.Lease.EnableIPv4 {
log.Info("Subnet removed: ", evt.Lease.Subnet)

if err := netlink.RouteDel(route); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
continue
route := n.GetRoute(&evt.Lease)
// Always remove the route from the route list.
n.removeFromV4RouteList(*route)

if err := netlink.RouteDel(route); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.Subnet, err)
}
}

if evt.Lease.EnableIPv6 {
log.Info("Subnet removed: ", evt.Lease.IPv6Subnet)

route := n.GetV6Route(&evt.Lease)
// Always remove the route from the route list.
n.removeFromV6RouteList(*route)

if err := netlink.RouteDel(route); err != nil {
log.Errorf("Error deleting route to %v: %v", evt.Lease.IPv6Subnet, err)
}
}

default:
Expand All @@ -138,22 +139,74 @@ func (n *RouteNetwork) handleSubnetEvents(batch []subnet.Event) {
}
}

func (n *RouteNetwork) addToRouteList(route netlink.Route) {
for _, r := range n.routes {
if routeEqual(r, route) {
func routeAdd(route *netlink.Route, ipFamily int, addToRouteList, removeFromRouteList func(netlink.Route)) {
addToRouteList(*route)
// Check if route exists before attempting to add it
routeList, err := netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
if err != nil {
log.Warningf("Unable to list routes: %v", err)
}

if len(routeList) > 0 && !routeEqual(routeList[0], *route) {
// Same Dst different Gw or different link index. Remove it, correct route will be added below.
log.Warningf("Replacing existing route to %v with %v", routeList[0], route)
if err := netlink.RouteDel(&routeList[0]); err != nil {
log.Errorf("Effor deleteing route to %v: %v", routeList[0].Dst, err)
return
}
removeFromRouteList(routeList[0])
}
routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
if err != nil {
log.Warningf("Unable to list routes: %v", err)
}

if len(routeList) > 0 && routeEqual(routeList[0], *route) {
// Same Dst and same Gw, keep it and do not attempt to add it.
log.Infof("Route to %v already exists, skipping.", route)
} else if err := netlink.RouteAdd(route); err != nil {
log.Errorf("Error adding route to %v", route)
return
}
routeList, err = netlink.RouteListFiltered(ipFamily, &netlink.Route{Dst: route.Dst}, netlink.RT_FILTER_DST)
if err != nil {
log.Warningf("Unable to list routes: %v", err)
}
}

func (n *RouteNetwork) addToRouteList(route netlink.Route) {
n.routes = addToRouteList(&route, n.routes)
}

func (n *RouteNetwork) addToV6RouteList(route netlink.Route) {
n.v6Routes = addToRouteList(&route, n.v6Routes)
}

func addToRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {
for _, r := range routes {
if routeEqual(r, *route) {
return routes
}
}
n.routes = append(n.routes, route)
return append(routes, *route)
}

func (n *RouteNetwork) removeFromRouteList(route netlink.Route) {
for index, r := range n.routes {
if routeEqual(r, route) {
n.routes = append(n.routes[:index], n.routes[index+1:]...)
return
func (n *RouteNetwork) removeFromV4RouteList(route netlink.Route) {
n.routes = n.removeFromRouteList(&route, n.routes)
}

func (n *RouteNetwork) removeFromV6RouteList(route netlink.Route) {
n.v6Routes = n.removeFromRouteList(&route, n.v6Routes)
}

func (n *RouteNetwork) removeFromRouteList(route *netlink.Route, routes []netlink.Route) []netlink.Route {
for index, r := range routes {
if routeEqual(r, *route) {
routes = append(routes[:index], routes[index+1:]...)
return routes
}
}
return routes
}

func (n *RouteNetwork) routeCheck(ctx context.Context) {
Expand All @@ -162,15 +215,24 @@ func (n *RouteNetwork) routeCheck(ctx context.Context) {
case <-ctx.Done():
return
case <-time.After(routeCheckRetries * time.Second):
n.checkSubnetExistInRoutes()
n.checkSubnetExistInV4Routes()
n.checkSubnetExistInV6Routes()
}
}
}

func (n *RouteNetwork) checkSubnetExistInRoutes() {
routeList, err := netlink.RouteList(nil, netlink.FAMILY_V4)
func (n *RouteNetwork) checkSubnetExistInV4Routes() {
n.checkSubnetExistInRoutes(n.routes, netlink.FAMILY_V4)
}

func (n *RouteNetwork) checkSubnetExistInV6Routes() {
n.checkSubnetExistInRoutes(n.v6Routes, netlink.FAMILY_V6)
}

func (n *RouteNetwork) checkSubnetExistInRoutes(routes []netlink.Route, ipFamily int) {
routeList, err := netlink.RouteList(nil, ipFamily)
if err == nil {
for _, route := range n.routes {
for _, route := range routes {
exist := false
for _, r := range routeList {
if r.Dst == nil {
Expand Down
59 changes: 57 additions & 2 deletions backend/route_network_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func TestRouteCache(t *testing.T) {
subnet1 := ip.IP4Net{IP: ip.FromIP(net.ParseIP("192.168.0.0")), PrefixLen: 24}
nw.handleSubnetEvents([]subnet.Event{
{Type: subnet.EventAdded, Lease: subnet.Lease{
Subnet: subnet1, Attrs: subnet.LeaseAttrs{PublicIP: gw1, BackendType: "host-gw"}}},
Subnet: subnet1, EnableIPv4: true, Attrs: subnet.LeaseAttrs{PublicIP: gw1, BackendType: "host-gw"}}},
})
if len(nw.routes) != 1 {
t.Fatal(nw.routes)
Expand All @@ -68,11 +68,66 @@ func TestRouteCache(t *testing.T) {
// change gateway of previous route
nw.handleSubnetEvents([]subnet.Event{
{Type: subnet.EventAdded, Lease: subnet.Lease{
Subnet: subnet1, Attrs: subnet.LeaseAttrs{PublicIP: gw2, BackendType: "host-gw"}}}})
Subnet: subnet1, EnableIPv4: true, Attrs: subnet.LeaseAttrs{PublicIP: gw2, BackendType: "host-gw"}}}})
if len(nw.routes) != 1 {
t.Fatal(nw.routes)
}
if !routeEqual(nw.routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw2.ToIP(), LinkIndex: lo.Attrs().Index}) {
t.Fatal(nw.routes[0])
}
}

func TestV6RouteCache(t *testing.T) {
teardown := ns.SetUpNetlinkTest(t)
defer teardown()

la := netlink.NewLinkAttrs()
la.Name = "br"
br := &netlink.Bridge{LinkAttrs: la}
if err := netlink.LinkAdd(br); err != nil {
t.Fatal(err)
}
if err := netlink.AddrAdd(br, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::1"), Mask: net.CIDRMask(64, 128)}}); err != nil {
t.Fatal(err)
}
if err := netlink.LinkSetUp(br); err != nil {
t.Fatal(err)
}

nw := RouteNetwork{
SimpleNetwork: SimpleNetwork{
ExtIface: &ExternalInterface{Iface: &net.Interface{Index: br.Attrs().Index}},
},
BackendType: "host-gw",
LinkIndex: br.Attrs().Index,
}
nw.GetV6Route = func(lease *subnet.Lease) *netlink.Route {
return &netlink.Route{
Dst: lease.IPv6Subnet.ToIPNet(),
Gw: lease.Attrs.PublicIPv6.ToIP(),
LinkIndex: nw.LinkIndex,
}
}
gw1, gw2 := ip.FromIP6(net.ParseIP("2001:db8:1::2")), ip.FromIP6(net.ParseIP("::2"))
subnet1 := ip.IP6Net{IP: ip.FromIP6(net.ParseIP("2001:db8:ffff::")), PrefixLen: 64}
nw.handleSubnetEvents([]subnet.Event{
{Type: subnet.EventAdded, Lease: subnet.Lease{
IPv6Subnet: subnet1, EnableIPv6: true, Attrs: subnet.LeaseAttrs{PublicIPv6: gw1, BackendType: "host-gw"}}},
})
if len(nw.v6Routes) != 1 {
t.Fatal(nw.v6Routes)
}
if !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw1.ToIP(), LinkIndex: br.Attrs().Index}) {
t.Fatal(nw.v6Routes[0])
}
// change gateway of previous route
nw.handleSubnetEvents([]subnet.Event{
{Type: subnet.EventAdded, Lease: subnet.Lease{
IPv6Subnet: subnet1, EnableIPv6: true, Attrs: subnet.LeaseAttrs{PublicIPv6: gw2, BackendType: "host-gw"}}}})
if len(nw.v6Routes) != 1 {
t.Fatal(nw.v6Routes)
}
if !routeEqual(nw.v6Routes[0], netlink.Route{Dst: subnet1.ToIPNet(), Gw: gw2.ToIP(), LinkIndex: br.Attrs().Index}) {
t.Fatal(nw.v6Routes[0])
}
}
19 changes: 10 additions & 9 deletions subnet/kube/kube.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,14 +277,15 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
n.Annotations[ksm.annotations.BackendPublicIP] != attrs.PublicIP.String() ||
n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
(n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != attrs.PublicIP.String())) ||
(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||
n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||
n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String())) {
(attrs.PublicIPv6 != nil &&
(n.Annotations[ksm.annotations.BackendV6Data] != string(v6Bd) ||
n.Annotations[ksm.annotations.BackendType] != attrs.BackendType ||
n.Annotations[ksm.annotations.BackendPublicIPv6] != attrs.PublicIPv6.String() ||
n.Annotations[ksm.annotations.SubnetKubeManaged] != "true" ||
(n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" && n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != attrs.PublicIPv6.String()))) {
n.Annotations[ksm.annotations.BackendType] = attrs.BackendType

//TODO -i only vxlan backend support dual stack now.
//TODO -i only vxlan and host-gw backends support dual stack now.
if (attrs.BackendType == "vxlan" && string(bd) != "null") || attrs.BackendType != "vxlan" {
n.Annotations[ksm.annotations.BackendData] = string(bd)
if n.Annotations[ksm.annotations.BackendPublicIPOverwrite] != "" {
Expand All @@ -299,7 +300,7 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
}
}

if string(v6Bd) != "null" {
if (attrs.BackendType == "vxlan" && string(v6Bd) != "null") || (attrs.BackendType == "host-gw" && attrs.PublicIPv6 != nil) {
n.Annotations[ksm.annotations.BackendV6Data] = string(v6Bd)
if n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] != "" {
if n.Annotations[ksm.annotations.BackendPublicIPv6] != n.Annotations[ksm.annotations.BackendPublicIPv6Overwrite] {
Expand Down Expand Up @@ -354,8 +355,8 @@ func (ksm *kubeSubnetManager) AcquireLease(ctx context.Context, attrs *subnet.Le
if ipv6Cidr != nil {
lease.IPv6Subnet = ip.FromIP6Net(ipv6Cidr)
}
//TODO - only vxlan backend support dual stack now.
if attrs.BackendType != "vxlan" {
//TODO - only vxlan and host-gw backends support dual stack now.
if attrs.BackendType != "vxlan" && attrs.BackendType != "host-gw" {
lease.EnableIPv4 = true
lease.EnableIPv6 = false
}
Expand Down

0 comments on commit 2859fd8

Please sign in to comment.