From 29feb042098d72abd9e544870a1c41c658fa594e Mon Sep 17 00:00:00 2001 From: "Johannes M. Scheuermann" Date: Tue, 17 Sep 2019 23:36:20 +0200 Subject: [PATCH 1/2] Update cni spec version to 0.3.1 to support multiple ips --- pkg/kubelet/dockershim/docker_service.go | 2 +- pkg/kubelet/dockershim/network/kubenet/BUILD | 4 +- .../network/kubenet/kubenet_linux.go | 52 ++++++++----------- 3 files changed, 26 insertions(+), 32 deletions(-) diff --git a/pkg/kubelet/dockershim/docker_service.go b/pkg/kubelet/dockershim/docker_service.go index 227bef8211a5..b7e67ca695fa 100644 --- a/pkg/kubelet/dockershim/docker_service.go +++ b/pkg/kubelet/dockershim/docker_service.go @@ -112,7 +112,7 @@ type NetworkPluginSettings struct { NonMasqueradeCIDR string // PluginName is the name of the plugin, runtime shim probes for PluginName string - // PluginBinDirString is a list of directiores delimited by commas, in + // PluginBinDirString is a list of directories delimited by commas, in // which the binaries for the plugin with PluginName may be found. PluginBinDirString string // PluginBinDirs is an array of directories in which the binaries for diff --git a/pkg/kubelet/dockershim/network/kubenet/BUILD b/pkg/kubelet/dockershim/network/kubenet/BUILD index 358c95116d6f..0abd1e22e667 100644 --- a/pkg/kubelet/dockershim/network/kubenet/BUILD +++ b/pkg/kubelet/dockershim/network/kubenet/BUILD @@ -31,7 +31,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/github.com/containernetworking/cni/libcni:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", - "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library", + "//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library", "//vendor/github.com/vishvananda/netlink:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog:go_default_library", @@ -74,7 +74,7 @@ go_library( "//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library", "//vendor/github.com/containernetworking/cni/libcni:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", - "//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library", + "//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library", "//vendor/github.com/vishvananda/netlink:go_default_library", "//vendor/golang.org/x/sys/unix:go_default_library", "//vendor/k8s.io/klog:go_default_library", diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index bf7a828ec2db..b7b23f62e9d3 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -29,7 +29,7 @@ import ( "github.com/containernetworking/cni/libcni" cnitypes "github.com/containernetworking/cni/pkg/types" - cnitypes020 "github.com/containernetworking/cni/pkg/types/020" + cnitypescurrent "github.com/containernetworking/cni/pkg/types/current" "github.com/vishvananda/netlink" "golang.org/x/sys/unix" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -67,13 +67,12 @@ const ( zeroCIDRv4 = "0.0.0.0/0" NET_CONFIG_TEMPLATE = `{ - "cniVersion": "0.1.0", + "cniVersion": "0.3.1", "name": "kubenet", "type": "bridge", "bridge": "%s", "mtu": %d, - "addIf": "%s", - "isGateway": true, + "isDefaultGateway": true, "ipMasq": false, "hairpinMode": %t, "ipam": { @@ -192,7 +191,7 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletc return nil } -// TODO: move thic logic into cni bridge plugin and remove this from kubenet +// TODO: move this logic into cni bridge plugin and remove this from kubenet func (plugin *kubenetNetworkPlugin) ensureMasqRule() error { if plugin.nonMasqueradeCIDR != zeroCIDRv4 && plugin.nonMasqueradeCIDR != zeroCIDRv6 { // switch according to target nonMasqueradeCidr ip family @@ -278,7 +277,7 @@ func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interf //setup hairpinMode setHairpin := plugin.hairpinMode == kubeletconfig.HairpinVeth - json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.mtu, network.DefaultInterfaceName, setHairpin, plugin.getRangesConfig(), plugin.getRoutesConfig()) + json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.mtu, setHairpin, plugin.getRangesConfig(), plugin.getRoutesConfig()) klog.V(4).Infof("CNI network config set to %v", json) plugin.netConfig, err = libcni.ConfFromBytes([]byte(json)) if err != nil { @@ -331,7 +330,6 @@ func (plugin *kubenetNetworkPlugin) Capabilities() utilsets.Int { // setup sets up networking through CNI using the given ns/name and sandbox ID. func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { - var ipv4, ipv6 net.IP var podGateways []net.IP var podCIDRs []net.IPNet @@ -351,27 +349,31 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube return err } // Coerce the CNI result version - res, err := cnitypes020.GetResult(resT) + res, err := cnitypescurrent.GetResult(resT) if err != nil { return fmt.Errorf("unable to understand network config: %v", err) } - //TODO: v1.16 (khenidak) update NET_CONFIG_TEMPLATE to CNI version 0.3.0 or later so - // that we get multiple IP addresses in the returned Result structure - if res.IP4 != nil { - ipv4 = res.IP4.IP.IP.To4() - podGateways = append(podGateways, res.IP4.Gateway) - podCIDRs = append(podCIDRs, net.IPNet{IP: ipv4.Mask(res.IP4.IP.Mask), Mask: res.IP4.IP.Mask}) - } - if res.IP6 != nil { - ipv6 = res.IP6.IP.IP - podGateways = append(podGateways, res.IP6.Gateway) - podCIDRs = append(podCIDRs, net.IPNet{IP: ipv6.Mask(res.IP6.IP.Mask), Mask: res.IP6.IP.Mask}) + if len(res.IPs) == 0 { + return fmt.Errorf("cni didn't report any IPs") } - if ipv4 == nil && ipv6 == nil { - return fmt.Errorf("cni didn't report ipv4 ipv6") + // Logic taken from: https://github.com/kubernetes/kubernetes/pull/69821 + for _, ip := range res.IPs { + if ip.Interface == nil || *ip.Interface > len(res.Interfaces) { + continue + } + + intf := res.Interfaces[*ip.Interface] + if intf.Sandbox != "" && intf.Name == network.DefaultInterfaceName { + // Found our sandbox interface, use the IP + podGateways = append(podGateways, ip.Gateway) + podCIDRs = append(podCIDRs, net.IPNet{IP: ip.Address.IP.Mask(ip.Address.Mask), Mask: ip.Address.Mask}) + // Add the IP to tracked IPs + plugin.addPodIP(id, ip.Address.IP.String()) + } } + // Put the container bridge into promiscuous mode to force it to accept hairpin packets. // TODO: Remove this once the kernel bug (#20096) is fixed. if plugin.hairpinMode == kubeletconfig.PromiscuousBridge { @@ -391,14 +393,6 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube plugin.syncEbtablesDedupRules(link.Attrs().HardwareAddr, podCIDRs, podGateways) } - // add the ip to tracked ips - if ipv4 != nil { - plugin.addPodIP(id, ipv4.String()) - } - if ipv6 != nil { - plugin.addPodIP(id, ipv6.String()) - } - if err := plugin.addTrafficShaping(id, annotations); err != nil { return err } From 8545dcc16141c771fdb81d0a2c7a13af14fca955 Mon Sep 17 00:00:00 2001 From: "Johannes M. Scheuermann" Date: Thu, 26 Mar 2020 20:41:14 +0100 Subject: [PATCH 2/2] Add unit tests --- pkg/kubelet/dockershim/network/kubenet/BUILD | 2 + .../network/kubenet/kubenet_linux.go | 41 ++++--- .../network/kubenet/kubenet_linux_test.go | 112 ++++++++++++++++++ 3 files changed, 138 insertions(+), 17 deletions(-) diff --git a/pkg/kubelet/dockershim/network/kubenet/BUILD b/pkg/kubelet/dockershim/network/kubenet/BUILD index 0abd1e22e667..a975a9a23d3d 100644 --- a/pkg/kubelet/dockershim/network/kubenet/BUILD +++ b/pkg/kubelet/dockershim/network/kubenet/BUILD @@ -133,6 +133,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/containernetworking/cni/libcni:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", + "//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/mock:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", @@ -151,6 +152,7 @@ go_test( "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//vendor/github.com/containernetworking/cni/libcni:go_default_library", "//vendor/github.com/containernetworking/cni/pkg/types:go_default_library", + "//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/mock:go_default_library", "//vendor/k8s.io/utils/exec:go_default_library", diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go index b7b23f62e9d3..7b509eb1f796 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go @@ -328,11 +328,31 @@ func (plugin *kubenetNetworkPlugin) Capabilities() utilsets.Int { return utilsets.NewInt() } -// setup sets up networking through CNI using the given ns/name and sandbox ID. -func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { +// adds IP addresses to the plugin and return the Pods gateway and cidr +func (plugin *kubenetNetworkPlugin) trackPodIPs(id kubecontainer.ContainerID, res *cnitypescurrent.Result) ([]net.IP, []net.IPNet) { var podGateways []net.IP var podCIDRs []net.IPNet + // Logic taken from: https://github.com/kubernetes/kubernetes/pull/69821 + for _, ip := range res.IPs { + if ip.Interface == nil || *ip.Interface > len(res.Interfaces) { + continue + } + + intf := res.Interfaces[*ip.Interface] + if intf.Sandbox != "" && intf.Name == network.DefaultInterfaceName { + // Found our sandbox interface, use the IP + podGateways = append(podGateways, ip.Gateway) + podCIDRs = append(podCIDRs, net.IPNet{IP: ip.Address.IP.Mask(ip.Address.Mask), Mask: ip.Address.Mask}) + plugin.addPodIP(id, ip.Address.IP.String()) + } + } + + return podGateways, podCIDRs +} + +// setup sets up networking through CNI using the given ns/name and sandbox ID. +func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error { // Disable DAD so we skip the kernel delay on bringing up new interfaces. if err := plugin.disableContainerDAD(id); err != nil { klog.V(3).Infof("Failed to disable DAD in container: %v", err) @@ -358,21 +378,8 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube return fmt.Errorf("cni didn't report any IPs") } - // Logic taken from: https://github.com/kubernetes/kubernetes/pull/69821 - for _, ip := range res.IPs { - if ip.Interface == nil || *ip.Interface > len(res.Interfaces) { - continue - } - - intf := res.Interfaces[*ip.Interface] - if intf.Sandbox != "" && intf.Name == network.DefaultInterfaceName { - // Found our sandbox interface, use the IP - podGateways = append(podGateways, ip.Gateway) - podCIDRs = append(podCIDRs, net.IPNet{IP: ip.Address.IP.Mask(ip.Address.Mask), Mask: ip.Address.Mask}) - // Add the IP to tracked IPs - plugin.addPodIP(id, ip.Address.IP.String()) - } - } + // Add IPs to the plugin cache + podGateways, podCIDRs := plugin.trackPodIPs(id, res) // Put the container bridge into promiscuous mode to force it to accept hairpin packets. // TODO: Remove this once the kernel bug (#20096) is fixed. diff --git a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go index cedb7772b0c6..d307c772b8ca 100644 --- a/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go +++ b/pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go @@ -18,6 +18,8 @@ package kubenet import ( "fmt" + cnitypes "github.com/containernetworking/cni/pkg/types" + cnitypescurrent "github.com/containernetworking/cni/pkg/types/current" "net" "strings" "testing" @@ -387,4 +389,114 @@ func TestGetRangesConfig(t *testing.T) { } } +func TestTrackPodIPs(t *testing.T) { + ipv4Gateway := "192.168.0.1" + ipv4Address := "192.168.0.2/24" + ipv4, _ := cnitypes.ParseCIDR(ipv4Address) + _, ipv4Net, _ := net.ParseCIDR(ipv4Address) + + ipv6Gateway := "abcd:1234:ffff::1" + ipv6Address := "abcd:1234:ffff::cdde/64" + ipv6, _ := cnitypes.ParseCIDR(ipv6Address) + _, ipv6Net, _ := net.ParseCIDR(ipv6Address) + + ipv4conf := &cnitypescurrent.IPConfig{ + Version: "4", + Interface: cnitypescurrent.Int(0), + Address: *ipv4, + Gateway: net.ParseIP(ipv4Gateway), + } + ipv6conf := &cnitypescurrent.IPConfig{ + Version: "6", + Interface: cnitypescurrent.Int(0), + Address: *ipv6, + Gateway: net.ParseIP(ipv6Gateway), + } + badconf := &cnitypescurrent.IPConfig{ + Version: "6", + Interface: nil, + Address: *ipv6, + Gateway: net.ParseIP(ipv6Gateway), + } + + for _, test := range []struct { + cniResult *cnitypescurrent.Result + expectedPodGateways []net.IP + expectedPodCIDRs []net.IPNet + expectedIPs []string + }{ + { + &cnitypescurrent.Result{ + CNIVersion: "0.3.1", + Interfaces: []*cnitypescurrent.Interface{ + { + Name: network.DefaultInterfaceName, + Mac: "00:11:22:33:44:55", + Sandbox: "/proc/1234/ns/net", + }, + }, + IPs: []*cnitypescurrent.IPConfig{ipv4conf}, + }, + []net.IP{net.ParseIP(ipv4Gateway)}, + []net.IPNet{*ipv4Net}, + []string{"192.168.0.2"}, + }, + { + &cnitypescurrent.Result{ + CNIVersion: "0.3.1", + Interfaces: []*cnitypescurrent.Interface{ + { + Name: network.DefaultInterfaceName, + Mac: "00:11:22:33:44:55", + Sandbox: "/proc/1234/ns/net", + }, + }, + IPs: []*cnitypescurrent.IPConfig{ipv6conf}, + }, + []net.IP{net.ParseIP(ipv6Gateway)}, + []net.IPNet{*ipv6Net}, + []string{"abcd:1234:ffff::cdde"}, + }, + { + &cnitypescurrent.Result{ + CNIVersion: "0.3.1", + Interfaces: []*cnitypescurrent.Interface{ + { + Name: network.DefaultInterfaceName, + Mac: "00:11:22:33:44:55", + Sandbox: "/proc/1234/ns/net", + }, + }, + IPs: []*cnitypescurrent.IPConfig{ipv4conf, ipv6conf}, + }, + []net.IP{net.ParseIP(ipv4Gateway), net.ParseIP(ipv6Gateway)}, + []net.IPNet{*ipv4Net, *ipv6Net}, + []string{"192.168.0.2", "abcd:1234:ffff::cdde"}, + }, + { + &cnitypescurrent.Result{ + CNIVersion: "0.3.1", + Interfaces: []*cnitypescurrent.Interface{}, + IPs: []*cnitypescurrent.IPConfig{badconf}, + }, + *new([]net.IP), + *new([]net.IPNet), + *new([]string), + }, + } { + fakeKubenet := &kubenetNetworkPlugin{podCIDRs: []*net.IPNet{ipv4}, podIPs: map[kubecontainer.ContainerID]utilsets.String{}} + testID := kubecontainer.ContainerID{ + Type: "fake", + ID: "1234", + } + podGateways, podCIDRs := fakeKubenet.trackPodIPs(testID, test.cniResult) + + cachedIPs, _ := fakeKubenet.getCachedPodIPs(testID) + + assert.Equal(t, cachedIPs, test.expectedIPs) + assert.Equal(t, podCIDRs, test.expectedPodCIDRs) + assert.Equal(t, podGateways, test.expectedPodGateways) + } +} + //TODO: add unit test for each implementation of network plugin interface