Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding traffic shaping support for CNI network driver #63194

Merged
merged 2 commits into from Jul 9, 2018
Merged
Changes from 1 commit
Commits
File filter...
Filter file types
Jump to…
Jump to file or symbol
Failed to load files and symbols.
+44 −9
Diff settings

Always

Just for now

Next

add traffic shaping support for CNI network driver

  • Loading branch information...
m1093782566 committed Apr 26, 2018
commit 8038a0dfa65244f54d0c362fe76e7ff224cbe801
@@ -51,6 +51,7 @@ go_library(
"//pkg/kubelet/apis/kubeletconfig:go_default_library",
"//pkg/kubelet/container:go_default_library",
"//pkg/kubelet/dockershim/network:go_default_library",
"//pkg/util/bandwidth:go_default_library",
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
@@ -29,6 +29,7 @@ import (
"k8s.io/kubernetes/pkg/kubelet/apis/kubeletconfig"
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
"k8s.io/kubernetes/pkg/kubelet/dockershim/network"
"k8s.io/kubernetes/pkg/util/bandwidth"
utilexec "k8s.io/utils/exec"
)

@@ -68,6 +69,22 @@ type cniPortMapping struct {
HostIP string `json:"hostIP"`
}

// cniBandwidthEntry maps to the standard CNI bandwidth Capability
// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md and
// https://github.com/containernetworking/plugins/blob/master/plugins/meta/bandwidth/README.md
type cniBandwidthEntry struct {
// IngressRate is the bandwidth rate in bits per second for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set
IngressRate int `json:"ingressRate,omitempty"`
// IngressBurst is the bandwidth burst in bits for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set
// NOTE: it's not used for now and default to 0.
IngressBurst int `json:"ingressBurst,omitempty"`
// EgressRate is the bandwidth is the bandwidth rate in bits per second for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set
EgressRate int `json:"egressRate,omitempty"`
// EgressBurst is the bandwidth burst in bits for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set
// NOTE: it's not used for now and default to 0.
EgressBurst int `json:"egressBurst,omitempty"`
}

func SplitDirs(dirs string) []string {
// Use comma rather than colon to work better with Windows too
return strings.Split(dirs, ",")
@@ -217,13 +234,13 @@ func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubec

// Windows doesn't have loNetwork. It comes only with Linux
if plugin.loNetwork != nil {
if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath); err != nil {
if _, err = plugin.addToNetwork(plugin.loNetwork, name, namespace, id, netnsPath, annotations); err != nil {
glog.Errorf("Error while adding to cni lo network: %s", err)
return err
}
}

_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
_, err = plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations)
if err != nil {
glog.Errorf("Error while adding to cni network: %s", err)
return err
@@ -243,11 +260,11 @@ func (plugin *cniNetworkPlugin) TearDownPod(namespace string, name string, id ku
glog.Warningf("CNI failed to retrieve network namespace path: %v", err)
}

return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
return plugin.deleteFromNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil)
}

func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (cnitypes.Result, error) {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (cnitypes.Result, error) {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations)
if err != nil {
glog.Errorf("Error adding network when building cni runtime conf: %v", err)
return nil, err
@@ -264,8 +281,8 @@ func (plugin *cniNetworkPlugin) addToNetwork(network *cniNetwork, podName string
return res, nil
}

func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) error {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath)
func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) error {
rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations)
if err != nil {
glog.Errorf("Error deleting network when building cni runtime conf: %v", err)
return err
@@ -283,7 +300,7 @@ func (plugin *cniNetworkPlugin) deleteFromNetwork(network *cniNetwork, podName s
return nil
}

func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string) (*libcni.RuntimeConf, error) {
func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations map[string]string) (*libcni.RuntimeConf, error) {
glog.V(4).Infof("Got netns path %v", podNetnsPath)
glog.V(4).Infof("Using podns path %v", podNs)

@@ -321,5 +338,22 @@ func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string
"portMappings": portMappingsParam,
}

ingress, egress, err := bandwidth.ExtractPodBandwidthResources(annotations)
if err != nil {
return nil, fmt.Errorf("Error reading pod bandwidth annotations: %v", err)
}
if ingress != nil || egress != nil {
bandwidthParam := cniBandwidthEntry{}
if ingress != nil {
bandwidthParam.IngressRate = int(ingress.Value() / 1000)

This comment has been minimized.

Copy link
@squeed

squeed May 30, 2018

Contributor

The documentation was wrong; the units are bits per second.

This comment has been minimized.

Copy link
@m1093782566

m1093782566 May 31, 2018

Author Member

Done. Thanks!

bandwidthParam.IngressBurst = 0 // default to no limit
}
if egress != nil {
bandwidthParam.EgressRate = int(egress.Value() / 1000)

This comment has been minimized.

Copy link
@squeed

squeed May 30, 2018

Contributor

bits per second here

This comment has been minimized.

Copy link
@m1093782566

m1093782566 May 31, 2018

Author Member

Done.

bandwidthParam.EgressBurst = 0 // default to no limit
}
rt.CapabilityArgs["bandwidth"] = bandwidthParam
}

return rt, nil
}
@@ -42,7 +42,7 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin
return nil, fmt.Errorf("CNI failed to retrieve network namespace path: %v", err)
}

result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath)
result, err := plugin.addToNetwork(plugin.getDefaultNetwork(), name, namespace, id, netnsPath, nil)

glog.V(5).Infof("GetPodNetworkStatus result %+v", result)
if err != nil {
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.