Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update cni spec version to 0.3.1 to support multiple ips #82844

Closed
wants to merge 2 commits into from
Closed

Update cni spec version to 0.3.1 to support multiple ips #82844

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
29feb04
Update cni spec version to 0.3.1 to support multiple ips
johscheuer Sep 17, 2019
8545dcc
Add unit tests
johscheuer Mar 26, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/kubelet/dockershim/docker_service.go
View file
Open in desktop
Expand Up @@ -112,7 +112,7 @@ type NetworkPluginSettings struct {
NonMasqueradeCIDR string
// PluginName is the name of the plugin, runtime shim probes for
PluginName string
// PluginBinDirString is a list of directiores delimited by commas, in
// PluginBinDirString is a list of directories delimited by commas, in
// which the binaries for the plugin with PluginName may be found.
PluginBinDirString string
// PluginBinDirs is an array of directories in which the binaries for
Expand Down
6 changes: 4 additions & 2 deletions pkg/kubelet/dockershim/network/kubenet/BUILD
View file
Open in desktop
Expand Up @@ -31,7 +31,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library",
"//vendor/github.com/vishvananda/netlink:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
Expand Down Expand Up @@ -74,7 +74,7 @@ go_library(
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library",
"//vendor/github.com/vishvananda/netlink:go_default_library",
"//vendor/golang.org/x/sys/unix:go_default_library",
"//vendor/k8s.io/klog:go_default_library",
Expand Down Expand Up @@ -133,6 +133,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/mock:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
Expand All @@ -151,6 +152,7 @@ go_test(
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types/current:go_default_library",
"//vendor/github.com/stretchr/testify/assert:go_default_library",
"//vendor/github.com/stretchr/testify/mock:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
Expand Down
65 changes: 33 additions & 32 deletions pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go
View file
Open in desktop
Expand Up @@ -29,7 +29,7 @@ import (

"github.com/containernetworking/cni/libcni"
cnitypes "github.com/containernetworking/cni/pkg/types"
cnitypes020 "github.com/containernetworking/cni/pkg/types/020"
cnitypescurrent "github.com/containernetworking/cni/pkg/types/current"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
Expand Down Expand Up @@ -67,13 +67,12 @@ const (
zeroCIDRv4 = "0.0.0.0/0"

NET_CONFIG_TEMPLATE = `{
"cniVersion": "0.1.0",
"cniVersion": "0.3.1",
"name": "kubenet",
"type": "bridge",
"bridge": "%s",
"mtu": %d,
"addIf": "%s",
"isGateway": true,
"isDefaultGateway": true,
"ipMasq": false,
"hairpinMode": %t,
"ipam": {
Expand Down Expand Up @@ -192,7 +191,7 @@ func (plugin *kubenetNetworkPlugin) Init(host network.Host, hairpinMode kubeletc
return nil
}

// TODO: move thic logic into cni bridge plugin and remove this from kubenet
// TODO: move this logic into cni bridge plugin and remove this from kubenet
func (plugin *kubenetNetworkPlugin) ensureMasqRule() error {
if plugin.nonMasqueradeCIDR != zeroCIDRv4 && plugin.nonMasqueradeCIDR != zeroCIDRv6 {
// switch according to target nonMasqueradeCidr ip family
Expand Down Expand Up @@ -278,7 +277,7 @@ func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interf
//setup hairpinMode
setHairpin := plugin.hairpinMode == kubeletconfig.HairpinVeth

json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.mtu, network.DefaultInterfaceName, setHairpin, plugin.getRangesConfig(), plugin.getRoutesConfig())
json := fmt.Sprintf(NET_CONFIG_TEMPLATE, BridgeName, plugin.mtu, setHairpin, plugin.getRangesConfig(), plugin.getRoutesConfig())
klog.V(4).Infof("CNI network config set to %v", json)
plugin.netConfig, err = libcni.ConfFromBytes([]byte(json))
if err != nil {
Expand Down Expand Up @@ -329,12 +328,31 @@ func (plugin *kubenetNetworkPlugin) Capabilities() utilsets.Int {
return utilsets.NewInt()
}

// setup sets up networking through CNI using the given ns/name and sandbox ID.
func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
var ipv4, ipv6 net.IP
// adds IP addresses to the plugin and return the Pods gateway and cidr
func (plugin *kubenetNetworkPlugin) trackPodIPs(id kubecontainer.ContainerID, res *cnitypescurrent.Result) ([]net.IP, []net.IPNet) {
var podGateways []net.IP
var podCIDRs []net.IPNet

// Logic taken from: https://github.com/kubernetes/kubernetes/pull/69821
for _, ip := range res.IPs {
if ip.Interface == nil || *ip.Interface > len(res.Interfaces) {
continue
}

intf := res.Interfaces[*ip.Interface]
if intf.Sandbox != "" && intf.Name == network.DefaultInterfaceName {
// Found our sandbox interface, use the IP
podGateways = append(podGateways, ip.Gateway)
podCIDRs = append(podCIDRs, net.IPNet{IP: ip.Address.IP.Mask(ip.Address.Mask), Mask: ip.Address.Mask})
plugin.addPodIP(id, ip.Address.IP.String())
}
}

return podGateways, podCIDRs
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That will blank out PodIPs. Is this what we want?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The plugin.addPodIP is called directly in the for loop instead afterwards so the behaviour should be the same?

}

// setup sets up networking through CNI using the given ns/name and sandbox ID.
func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
// Disable DAD so we skip the kernel delay on bringing up new interfaces.
if err := plugin.disableContainerDAD(id); err != nil {
klog.V(3).Infof("Failed to disable DAD in container: %v", err)
Expand All @@ -351,27 +369,18 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube
return err
}
// Coerce the CNI result version
res, err := cnitypes020.GetResult(resT)
res, err := cnitypescurrent.GetResult(resT)
if err != nil {
return fmt.Errorf("unable to understand network config: %v", err)
}
//TODO: v1.16 (khenidak) update NET_CONFIG_TEMPLATE to CNI version 0.3.0 or later so
// that we get multiple IP addresses in the returned Result structure
if res.IP4 != nil {
ipv4 = res.IP4.IP.IP.To4()
podGateways = append(podGateways, res.IP4.Gateway)
podCIDRs = append(podCIDRs, net.IPNet{IP: ipv4.Mask(res.IP4.IP.Mask), Mask: res.IP4.IP.Mask})
}

if res.IP6 != nil {
ipv6 = res.IP6.IP.IP
podGateways = append(podGateways, res.IP6.Gateway)
podCIDRs = append(podCIDRs, net.IPNet{IP: ipv6.Mask(res.IP6.IP.Mask), Mask: res.IP6.IP.Mask})
if len(res.IPs) == 0 {
return fmt.Errorf("cni didn't report any IPs")
}

if ipv4 == nil && ipv6 == nil {
return fmt.Errorf("cni didn't report ipv4 ipv6")
}
// Add IPs to the plugin cache
podGateways, podCIDRs := plugin.trackPodIPs(id, res)

// Put the container bridge into promiscuous mode to force it to accept hairpin packets.
// TODO: Remove this once the kernel bug (#20096) is fixed.
if plugin.hairpinMode == kubeletconfig.PromiscuousBridge {
Expand All @@ -391,14 +400,6 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube
plugin.syncEbtablesDedupRules(link.Attrs().HardwareAddr, podCIDRs, podGateways)
}

// add the ip to tracked ips
if ipv4 != nil {
plugin.addPodIP(id, ipv4.String())
}
if ipv6 != nil {
plugin.addPodIP(id, ipv6.String())
}

if err := plugin.addTrafficShaping(id, annotations); err != nil {
return err
}
Expand Down
112 changes: 112 additions & 0 deletions pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go
View file
Open in desktop
Expand Up @@ -18,6 +18,8 @@ package kubenet

import (
"fmt"
cnitypes "github.com/containernetworking/cni/pkg/types"
cnitypescurrent "github.com/containernetworking/cni/pkg/types/current"
"net"
"strings"
"testing"
Expand Down Expand Up @@ -387,4 +389,114 @@ func TestGetRangesConfig(t *testing.T) {
}
}

func TestTrackPodIPs(t *testing.T) {
ipv4Gateway := "192.168.0.1"
ipv4Address := "192.168.0.2/24"
ipv4, _ := cnitypes.ParseCIDR(ipv4Address)
_, ipv4Net, _ := net.ParseCIDR(ipv4Address)

ipv6Gateway := "abcd:1234:ffff::1"
ipv6Address := "abcd:1234:ffff::cdde/64"
ipv6, _ := cnitypes.ParseCIDR(ipv6Address)
_, ipv6Net, _ := net.ParseCIDR(ipv6Address)

ipv4conf := &cnitypescurrent.IPConfig{
Version: "4",
Interface: cnitypescurrent.Int(0),
Address: *ipv4,
Gateway: net.ParseIP(ipv4Gateway),
}
ipv6conf := &cnitypescurrent.IPConfig{
Version: "6",
Interface: cnitypescurrent.Int(0),
Address: *ipv6,
Gateway: net.ParseIP(ipv6Gateway),
}
badconf := &cnitypescurrent.IPConfig{
Version: "6",
Interface: nil,
Address: *ipv6,
Gateway: net.ParseIP(ipv6Gateway),
}

for _, test := range []struct {
cniResult *cnitypescurrent.Result
expectedPodGateways []net.IP
expectedPodCIDRs []net.IPNet
expectedIPs []string
}{
{
&cnitypescurrent.Result{
CNIVersion: "0.3.1",
Interfaces: []*cnitypescurrent.Interface{
{
Name: network.DefaultInterfaceName,
Mac: "00:11:22:33:44:55",
Sandbox: "/proc/1234/ns/net",
},
},
IPs: []*cnitypescurrent.IPConfig{ipv4conf},
},
[]net.IP{net.ParseIP(ipv4Gateway)},
[]net.IPNet{*ipv4Net},
[]string{"192.168.0.2"},
},
{
&cnitypescurrent.Result{
CNIVersion: "0.3.1",
Interfaces: []*cnitypescurrent.Interface{
{
Name: network.DefaultInterfaceName,
Mac: "00:11:22:33:44:55",
Sandbox: "/proc/1234/ns/net",
},
},
IPs: []*cnitypescurrent.IPConfig{ipv6conf},
},
[]net.IP{net.ParseIP(ipv6Gateway)},
[]net.IPNet{*ipv6Net},
[]string{"abcd:1234:ffff::cdde"},
},
{
&cnitypescurrent.Result{
CNIVersion: "0.3.1",
Interfaces: []*cnitypescurrent.Interface{
{
Name: network.DefaultInterfaceName,
Mac: "00:11:22:33:44:55",
Sandbox: "/proc/1234/ns/net",
},
},
IPs: []*cnitypescurrent.IPConfig{ipv4conf, ipv6conf},
},
[]net.IP{net.ParseIP(ipv4Gateway), net.ParseIP(ipv6Gateway)},
[]net.IPNet{*ipv4Net, *ipv6Net},
[]string{"192.168.0.2", "abcd:1234:ffff::cdde"},
},
{
&cnitypescurrent.Result{
CNIVersion: "0.3.1",
Interfaces: []*cnitypescurrent.Interface{},
IPs: []*cnitypescurrent.IPConfig{badconf},
},
*new([]net.IP),
*new([]net.IPNet),
*new([]string),
},
} {
fakeKubenet := &kubenetNetworkPlugin{podCIDRs: []*net.IPNet{ipv4}, podIPs: map[kubecontainer.ContainerID]utilsets.String{}}
testID := kubecontainer.ContainerID{
Type: "fake",
ID: "1234",
}
podGateways, podCIDRs := fakeKubenet.trackPodIPs(testID, test.cniResult)

cachedIPs, _ := fakeKubenet.getCachedPodIPs(testID)

assert.Equal(t, cachedIPs, test.expectedIPs)
assert.Equal(t, podCIDRs, test.expectedPodCIDRs)
assert.Equal(t, podGateways, test.expectedPodGateways)
}
}

//TODO: add unit test for each implementation of network plugin interface