Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kubenet: replace gateway with cni result #85993

Merged
merged 1 commit into from Jan 10, 2020
Merged

kubenet: replace gateway with cni result #85993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
34 changes: 18 additions & 16 deletions pkg/kubelet/dockershim/network/kubenet/kubenet_linux.go
View file
Open in desktop
Expand Up @@ -118,7 +118,6 @@ type kubenetNetworkPlugin struct {
nonMasqueradeCIDR string
cacheDir string
podCIDRs []*net.IPNet
podGateways []net.IP
}

func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugin {
Expand All @@ -139,7 +138,6 @@ func NewPlugin(networkPluginDirs []string, cacheDir string) network.NetworkPlugi
nonMasqueradeCIDR: "10.0.0.0/8",
cacheDir: cacheDir,
podCIDRs: make([]*net.IPNet, 0),
podGateways: make([]net.IP, 0),
}
}

Expand Down Expand Up @@ -270,13 +268,11 @@ func (plugin *kubenetNetworkPlugin) Event(name string, details map[string]interf
for idx, currentPodCIDR := range podCIDRs {
_, cidr, err := net.ParseCIDR(currentPodCIDR)
if nil != err {
klog.Warningf("Failed to generate CNI network config with cidr %s at indx:%v: %v", currentPodCIDR, idx, err)
klog.Warningf("Failed to generate CNI network config with cidr %s at index:%v: %v", currentPodCIDR, idx, err)
return
}
// create list of ips and gateways
cidr.IP[len(cidr.IP)-1] += 1 // Set bridge address to first address in IPNet
// create list of ips
plugin.podCIDRs = append(plugin.podCIDRs, cidr)
plugin.podGateways = append(plugin.podGateways, cidr.IP)
}

//setup hairpinMode
Expand Down Expand Up @@ -336,6 +332,9 @@ func (plugin *kubenetNetworkPlugin) Capabilities() utilsets.Int {
// setup sets up networking through CNI using the given ns/name and sandbox ID.
func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kubecontainer.ContainerID, annotations map[string]string) error {
var ipv4, ipv6 net.IP
var podGateways []net.IP
var podCIDRs []net.IPNet

This comment was marked as resolved.

Sign in to view


// Disable DAD so we skip the kernel delay on bringing up new interfaces.
if err := plugin.disableContainerDAD(id); err != nil {
klog.V(3).Infof("Failed to disable DAD in container: %v", err)
Expand All @@ -360,10 +359,14 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube
// that we get multiple IP addresses in the returned Result structure
if res.IP4 != nil {
ipv4 = res.IP4.IP.IP.To4()
podGateways = append(podGateways, res.IP4.Gateway)
podCIDRs = append(podCIDRs, net.IPNet{IP: ipv4.Mask(res.IP4.IP.Mask), Mask: res.IP4.IP.Mask})
}

if res.IP6 != nil {
ipv6 = res.IP6.IP.IP
podGateways = append(podGateways, res.IP6.Gateway)
podCIDRs = append(podCIDRs, net.IPNet{IP: ipv6.Mask(res.IP6.IP.Mask), Mask: res.IP6.IP.Mask})
}

if ipv4 == nil && ipv6 == nil {
Expand All @@ -385,7 +388,7 @@ func (plugin *kubenetNetworkPlugin) setup(namespace string, name string, id kube
}

// configure the ebtables rules to eliminate duplicate packets by best effort
plugin.syncEbtablesDedupRules(link.Attrs().HardwareAddr)
plugin.syncEbtablesDedupRules(link.Attrs().HardwareAddr, podCIDRs, podGateways)
}

// add the ip to tracked ips
Expand Down Expand Up @@ -761,7 +764,7 @@ func (plugin *kubenetNetworkPlugin) shaper() bandwidth.Shaper {
}

//TODO: make this into a goroutine and rectify the dedup rules periodically
func (plugin *kubenetNetworkPlugin) syncEbtablesDedupRules(macAddr net.HardwareAddr) {
func (plugin *kubenetNetworkPlugin) syncEbtablesDedupRules(macAddr net.HardwareAddr, podCIDRs []net.IPNet, podGateways []net.IP) {
if plugin.ebtables == nil {
plugin.ebtables = utilebtables.New(plugin.execer)
klog.V(3).Infof("Flushing dedup chain")
Expand Down Expand Up @@ -790,8 +793,8 @@ func (plugin *kubenetNetworkPlugin) syncEbtablesDedupRules(macAddr net.HardwareA
}

// per gateway rule
for idx, gw := range plugin.podGateways {
klog.V(3).Infof("Filtering packets with ebtables on mac address: %v, gateway: %v, pod CIDR: %v", macAddr.String(), gw.String(), plugin.podCIDRs[idx].String())
for idx, gw := range podGateways {
klog.V(3).Infof("Filtering packets with ebtables on mac address: %v, gateway: %v, pod CIDR: %v", macAddr.String(), gw.String(), podCIDRs[idx].String())

bIsV6 := netutils.IsIPv6(gw)
IPFamily := "IPv4"
Expand All @@ -807,9 +810,9 @@ func (plugin *kubenetNetworkPlugin) syncEbtablesDedupRules(macAddr net.HardwareA
return

}
_, err = plugin.ebtables.EnsureRule(utilebtables.Append, utilebtables.TableFilter, dedupChain, append(commonArgs, ipSrc, plugin.podCIDRs[idx].String(), "-j", "DROP")...)
_, err = plugin.ebtables.EnsureRule(utilebtables.Append, utilebtables.TableFilter, dedupChain, append(commonArgs, ipSrc, podCIDRs[idx].String(), "-j", "DROP")...)
if err != nil {
klog.Errorf("Failed to ensure packets from podCidr[%v] but has mac address of cbr0 to get dropped. err:%v", plugin.podCIDRs[idx].String(), err)
klog.Errorf("Failed to ensure packets from podCidr[%v] but has mac address of cbr0 to get dropped. err:%v", podCIDRs[idx].String(), err)
return
}
}
Expand Down Expand Up @@ -861,18 +864,17 @@ func (plugin *kubenetNetworkPlugin) getRangesConfig() string {
createRange := func(thisNet *net.IPNet) string {
template := `
[{
"subnet": "%s",
"gateway": "%s"
"subnet": "%s"
}]`
return fmt.Sprintf(template, thisNet.String(), thisNet.IP.String())
return fmt.Sprintf(template, thisNet.String())
}

ranges := make([]string, len(plugin.podCIDRs))
for idx, thisCIDR := range plugin.podCIDRs {
ranges[idx] = createRange(thisCIDR)
}
//[{range}], [{range}]
// each range is a subnet and a gateway
// each range contains a subnet. gateway will be fetched from cni result
return strings.Join(ranges[:], ",")
}

Expand Down
75 changes: 57 additions & 18 deletions pkg/kubelet/dockershim/network/kubenet/kubenet_linux_test.go
View file
Open in desktop
Expand Up @@ -241,18 +241,28 @@ func TestInit_MTU(t *testing.T) {
func TestTearDownWithoutRuntime(t *testing.T) {
testCases := []struct {
podCIDR []string
expectedPodCIDR []string
ip string
expectedGateway []string
}{
{
podCIDR: []string{"10.0.0.0/24"},
expectedPodCIDR: []string{"10.0.0.0/24"},
ip: "10.0.0.1",
},
{
podCIDR: []string{"10.0.0.1/24"},
expectedPodCIDR: []string{"10.0.0.0/24"},
ip: "10.0.0.1",
expectedGateway: []string{"10.0.0.1"},
},
{
podCIDR: []string{"2001:beef::/48"},
expectedPodCIDR: []string{"2001:beef::/48"},
ip: "2001:beef::1",
},
{
podCIDR: []string{"2001:beef::1/48"},
expectedPodCIDR: []string{"2001:beef::/48"},
ip: "2001:beef::1",
expectedGateway: []string{"2001:beef::1"},
},
}
for _, tc := range testCases {
Expand Down Expand Up @@ -283,25 +293,13 @@ func TestTearDownWithoutRuntime(t *testing.T) {
details[network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE_DETAIL_CIDR] = strings.Join(tc.podCIDR, ",")
kubenet.Event(network.NET_PLUGIN_EVENT_POD_CIDR_CHANGE, details)

if len(kubenet.podGateways) != len(tc.expectedGateway) {
t.Errorf("generated gateway: %q, expecting: %q are not of the same length", kubenet.podGateways, tc.expectedGateway)
continue
}

for idx := range tc.expectedGateway {
if kubenet.podGateways[idx].String() != tc.expectedGateway[idx] {
t.Errorf("generated gateway: %q, expecting: %q", kubenet.podGateways[idx].String(), tc.expectedGateway[idx])

}
}

if len(kubenet.podCIDRs) != len(tc.podCIDR) {
t.Errorf("generated podCidr: %q, expecting: %q are not of the same length", kubenet.podCIDRs, tc.podCIDR)
continue
}
for idx := range tc.podCIDR {
if kubenet.podCIDRs[idx].String() != tc.podCIDR[idx] {
t.Errorf("generated podCidr: %q, expecting: %q", kubenet.podCIDRs[idx].String(), tc.podCIDR[idx])
if kubenet.podCIDRs[idx].String() != tc.expectedPodCIDR[idx] {
t.Errorf("generated podCidr: %q, expecting: %q", kubenet.podCIDRs[idx].String(), tc.expectedPodCIDR[idx])
}
}

Expand All @@ -319,7 +317,7 @@ func TestTearDownWithoutRuntime(t *testing.T) {
}
}

func TestGetRoutesConifg(t *testing.T) {
func TestGetRoutesConfig(t *testing.T) {
for _, test := range []struct {
cidrs []string
routes string
Expand Down Expand Up @@ -348,4 +346,45 @@ func TestGetRoutesConifg(t *testing.T) {
}
}

func TestGetRangesConfig(t *testing.T) {
for _, test := range []struct {
cidrs []string
ranges string
}{
{
cidrs: []string{"10.0.0.0/24"},
ranges: `
[{
"subnet": "10.0.0.0/24"
}]`,
},
{
cidrs: []string{"2001:4860::/32"},
ranges: `
[{
"subnet": "2001:4860::/32"
}]`,
},
{
cidrs: []string{"10.0.0.0/24", "2001:4860::/32"},
ranges: `
[{
"subnet": "10.0.0.0/24"
}],
[{
"subnet": "2001:4860::/32"
}]`,
},
} {
var cidrs []*net.IPNet
for _, c := range test.cidrs {
_, cidr, err := net.ParseCIDR(c)
assert.NoError(t, err)
cidrs = append(cidrs, cidr)
}
fakeKubenet := &kubenetNetworkPlugin{podCIDRs: cidrs}
assert.Equal(t, test.ranges, fakeKubenet.getRangesConfig())
}
}

//TODO: add unit test for each implementation of network plugin interface