-
Notifications
You must be signed in to change notification settings - Fork 123
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Improve bootstrap reliability on heterogeneous UPI network configurat…
…ions Before this change, bootstrap IP discovery assumed that the first address of the unicast interface must be the bootstrap IP. This assumption doesn't always hold in the face of user-defined interfaces and addresses whose ordering isn't guaranteed. When the assumptions are broken and the incorrect bootstrap IP is selected, bootstrapping fails because quorum cannot be established. This change improves the accuracy of bootstrap IP discovery by more flexibly accounting for a wider variety of possible network interface configurations. An IP is now considered the bootstrap IP if all of the following are true. For IPv4: * The IP is contained by the machine CIDR defined in the cluster configuration * On bare metal platforms, the IP is not the API or DNS VIP in the cluster configuration For IPv6, the same must be true in addition to the following: * The IP is not deprecated * The IP is routable according at least one non-default route This work is adapted from https://github.com/openshift/baremetal-runtimecfg/blob/master/pkg/utils/utils.go.
- Loading branch information
1 parent
e2b9445
commit 49bf483
Showing
82 changed files
with
16,042 additions
and
223 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,198 @@ | ||
| package render | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "net" | ||
|
|
||
| "github.com/vishvananda/netlink" | ||
| "k8s.io/klog" | ||
| ) | ||
|
|
||
| var getBootstrapIP = getBootstrapIPFromNetlink | ||
|
|
||
| // getBootstrapIPFromNetlink discovers the routable bootstrap node IP contained | ||
| // in machineCIDR using the native netlink library. Returns an error if no such | ||
| // IP could be found. | ||
| func getBootstrapIPFromNetlink(ipv6 bool, machineCIDR string, excludedIPs []string) (net.IP, error) { | ||
| ips, err := ipAddrs() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| addrMap, err := getAddrMap() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| routeMap, err := getRouteMap() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| addressFilter := AddressFilters( | ||
| NonDeprecatedAddress, | ||
| ContainedByCIDR(machineCIDR), | ||
| AddressNotIn(excludedIPs...), | ||
| ) | ||
| addresses, err := routableAddresses(addrMap, routeMap, ips, addressFilter, NonDefaultRoute) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| if len(addresses) == 0 { | ||
| return nil, fmt.Errorf("couldn't find any bootstrap node IPs") | ||
| } | ||
| if len(addresses) > 1 { | ||
| klog.Warningf("found multiple candidate bootstrap IPs; using only the first one: %+v", addresses) | ||
| } | ||
|
|
||
| var bootstrapIP net.IP | ||
| for _, ip := range addresses { | ||
| // IPv6 | ||
| if ipv6 && ip.To4() == nil { | ||
| bootstrapIP = ip | ||
| break | ||
| } | ||
| // IPv4 | ||
| if !ipv6 && ip.To4() != nil { | ||
| bootstrapIP = ip | ||
| break | ||
| } | ||
| } | ||
| if bootstrapIP == nil { | ||
| return nil, fmt.Errorf("couldn't find a suitable bootstrap node IP from candidates: %+v", addresses) | ||
| } | ||
|
|
||
| return bootstrapIP, nil | ||
| } | ||
|
|
||
| // AddressFilter is a function type to filter addresses | ||
| type AddressFilter func(netlink.Addr) bool | ||
|
|
||
| // RouteFilter is a function type to filter routes | ||
| type RouteFilter func(netlink.Route) bool | ||
|
|
||
| // NonDeprecatedAddress returns true if the address is IPv6 and has a preferred lifetime of 0 | ||
| func NonDeprecatedAddress(addr netlink.Addr) bool { | ||
| return !(net.IPv6len == len(addr.IP) && addr.PreferedLft == 0) | ||
| } | ||
|
|
||
| // NonDefaultRoute returns whether the passed Route is the default | ||
| func NonDefaultRoute(route netlink.Route) bool { | ||
| return route.Dst != nil | ||
| } | ||
|
|
||
| func ContainedByCIDR(cidr string) AddressFilter { | ||
| return func(addr netlink.Addr) bool { | ||
| _, parsedNet, err := net.ParseCIDR(cidr) | ||
| if err != nil { | ||
| panic(err) | ||
| } | ||
| return parsedNet.Contains(addr.IP) | ||
| } | ||
| } | ||
|
|
||
| func AddressNotIn(ips ...string) AddressFilter { | ||
| return func(addr netlink.Addr) bool { | ||
| for _, ip := range ips { | ||
| if addr.IP.String() == ip { | ||
| return false | ||
| } | ||
| } | ||
| return true | ||
| } | ||
| } | ||
|
|
||
| func AddressFilters(filters ...AddressFilter) AddressFilter { | ||
| return func(addr netlink.Addr) bool { | ||
| for _, include := range filters { | ||
| if !include(addr) { | ||
| return false | ||
| } | ||
| } | ||
| return true | ||
| } | ||
| } | ||
|
|
||
| type addrMap map[netlink.Link][]netlink.Addr | ||
| type routeMap map[int][]netlink.Route | ||
|
|
||
| // routableAddresses takes a slice of Virtual IPs and returns a slice of | ||
| // configured addresses in the current network namespace that directly route to | ||
| // those vips. You can optionally pass an AddressFilter and/or RouteFilter to | ||
| // further filter down which addresses are considered. | ||
| // | ||
| // This is ported from https://github.com/openshift/baremetal-runtimecfg/blob/master/pkg/utils/utils.go. | ||
| func routableAddresses(addrMap addrMap, routeMap routeMap, vips []net.IP, af AddressFilter, rf RouteFilter) ([]net.IP, error) { | ||
| matches := map[string]net.IP{} | ||
| for link, addresses := range addrMap { | ||
| for _, address := range addresses { | ||
| maskPrefix, maskBits := address.Mask.Size() | ||
| if !af(address) { | ||
| klog.Infof("Filtered address %+v", address) | ||
| continue | ||
| } | ||
| if net.IPv6len == len(address.IP) && maskPrefix == maskBits { | ||
| routes, ok := routeMap[link.Attrs().Index] | ||
| if !ok { | ||
| continue | ||
| } | ||
| for _, route := range routes { | ||
| if !rf(route) { | ||
| klog.Infof("Filtered route %+v for address %+v", route, address) | ||
| continue | ||
| } | ||
| routePrefix, _ := route.Dst.Mask.Size() | ||
| klog.Infof("Checking route %+v (mask %s) for address %+v", route, route.Dst.Mask, address) | ||
| if routePrefix == 0 { | ||
| continue | ||
| } | ||
| containmentNet := net.IPNet{IP: address.IP, Mask: route.Dst.Mask} | ||
| for _, vip := range vips { | ||
| klog.Infof("Checking whether address %s with route %s contains VIP %s", address, route, vip) | ||
| if containmentNet.Contains(vip) { | ||
| klog.Infof("Address %s with route %s contains VIP %s", address, route, vip) | ||
| matches[address.IP.String()] = address.IP | ||
| } | ||
| } | ||
| } | ||
| } else { | ||
| for _, vip := range vips { | ||
| klog.Infof("Checking whether address %s contains VIP %s", address, vip) | ||
| if address.Contains(vip) { | ||
| klog.Infof("Address %s contains VIP %s", address, vip) | ||
| matches[address.IP.String()] = address.IP | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| ips := []net.IP{} | ||
| for _, ip := range matches { | ||
| ips = append(ips, ip) | ||
| } | ||
| klog.Infof("Found routable IPs %+v", ips) | ||
| return ips, nil | ||
| } | ||
|
|
||
| func ipAddrs() ([]net.IP, error) { | ||
| ips := []net.IP{} | ||
| addrs, err := net.InterfaceAddrs() | ||
| if err != nil { | ||
| return ips, err | ||
| } | ||
| for _, addr := range addrs { | ||
| var ip net.IP | ||
| switch v := addr.(type) { | ||
| case *net.IPNet: | ||
| ip = v.IP | ||
| case *net.IPAddr: | ||
| ip = v.IP | ||
| } | ||
| if ip == nil { | ||
| continue | ||
| } | ||
| if !ip.IsGlobalUnicast() { | ||
| continue // we only want global unicast address | ||
| } | ||
| ips = append(ips, ip) | ||
| } | ||
| return ips, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| // +build !linux | ||
|
|
||
| package render | ||
|
|
||
| import ( | ||
| "fmt" | ||
| ) | ||
|
|
||
| func getAddrMap() (addrMap addrMap, err error) { | ||
| return nil, fmt.Errorf("not implemented") | ||
| } | ||
|
|
||
| func getRouteMap() (routeMap routeMap, err error) { | ||
| return nil, fmt.Errorf("not implemented") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package render | ||
|
|
||
| import ( | ||
| "github.com/vishvananda/netlink" | ||
| "golang.org/x/sys/unix" | ||
| "k8s.io/klog" | ||
| ) | ||
|
|
||
| func getAddrMap() (addrMap addrMap, err error) { | ||
| nlHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer nlHandle.Delete() | ||
|
|
||
| links, err := nlHandle.LinkList() | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| addrMap = make(map[netlink.Link][]netlink.Addr) | ||
| for _, link := range links { | ||
| addresses, err := nlHandle.AddrList(link, netlink.FAMILY_ALL) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| for _, address := range addresses { | ||
| if _, ok := addrMap[link]; ok { | ||
| addrMap[link] = append(addrMap[link], address) | ||
| } else { | ||
| addrMap[link] = []netlink.Addr{address} | ||
| } | ||
| } | ||
| } | ||
| klog.Infof("retrieved Address map %+v", addrMap) | ||
| return addrMap, nil | ||
| } | ||
|
|
||
| func getRouteMap() (routeMap routeMap, err error) { | ||
| nlHandle, err := netlink.NewHandle(unix.NETLINK_ROUTE) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| defer nlHandle.Delete() | ||
|
|
||
| routes, err := nlHandle.RouteList(nil, netlink.FAMILY_V6) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| routeMap = make(map[int][]netlink.Route) | ||
| for _, route := range routes { | ||
| if route.Protocol != unix.RTPROT_RA { | ||
| klog.Infof("Ignoring route non Router advertisement route %+v", route) | ||
| continue | ||
| } | ||
| if _, ok := routeMap[route.LinkIndex]; ok { | ||
| routeMap[route.LinkIndex] = append(routeMap[route.LinkIndex], route) | ||
| } else { | ||
| routeMap[route.LinkIndex] = []netlink.Route{route} | ||
| } | ||
| } | ||
|
|
||
| klog.Infof("Retrieved route map %+v", routeMap) | ||
|
|
||
| return routeMap, nil | ||
| } |
Oops, something went wrong.