Skip to content

Commit

Permalink
add netem qos when create pod (#1510)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongzhen-ma
hongzhen-ma committed May 10, 2022
1 parent 5158dd9 commit 5160362
Show file tree
Hide file tree
Showing 4 changed files with 78 additions and 8 deletions.
9 changes: 6 additions & 3 deletions pkg/daemon/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
}

var gatewayCheckMode int
var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, providerNetwork, ifName, nicType, podNicName, priority, vmName string
var macAddr, ip, ipAddr, cidr, gw, subnet, ingress, egress, providerNetwork, ifName, nicType, podNicName, priority, vmName, latency, limit, loss string
var isDefaultRoute bool
var pod *v1.Pod
var err error
Expand Down Expand Up @@ -120,6 +120,9 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
ingress = pod.Annotations[fmt.Sprintf(util.IngressRateAnnotationTemplate, podRequest.Provider)]
egress = pod.Annotations[fmt.Sprintf(util.EgressRateAnnotationTemplate, podRequest.Provider)]
priority = pod.Annotations[fmt.Sprintf(util.PriorityAnnotationTemplate, podRequest.Provider)]
latency = pod.Annotations[fmt.Sprintf(util.NetemQosLatencyAnnotationTemplate, podRequest.Provider)]
limit = pod.Annotations[fmt.Sprintf(util.NetemQosLimitAnnotationTemplate, podRequest.Provider)]
loss = pod.Annotations[fmt.Sprintf(util.NetemQosLossAnnotationTemplate, podRequest.Provider)]
providerNetwork = pod.Annotations[fmt.Sprintf(util.ProviderNetworkTemplate, podRequest.Provider)]
vmName = pod.Annotations[fmt.Sprintf(util.VmTemplate, podRequest.Provider)]
ipAddr = util.GetIpAddrWithMask(ip, cidr)
Expand Down Expand Up @@ -239,12 +242,12 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon

klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, podRequest.Routes)
if nicType == util.InternalType {
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode)
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, priority, podRequest.DeviceID, nicType, latency, limit, loss, gatewayCheckMode)
} else if nicType == util.DpdkType {
err = csh.configureDpdkNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, priority, getShortSharedDir(pod.UID, podRequest.VhostUserSocketVolumeName), podRequest.VhostUserSocketName)
} else {
podNicName = ifName
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, priority, podRequest.DeviceID, nicType, gatewayCheckMode)
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, podRequest.Routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, priority, podRequest.DeviceID, nicType, latency, limit, loss, gatewayCheckMode)
}
if err != nil {
errMsg := fmt.Errorf("configure nic failed %v", err)
Expand Down
12 changes: 10 additions & 2 deletions pkg/daemon/ovs_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, ne
return nil
}

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) error {
func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType, latency, limit, loss string, gwCheckMode int) error {
var err error
var hostNicName, containerNicName string
if DeviceID == "" {
Expand Down Expand Up @@ -98,6 +98,10 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
return err
}

if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss); err != nil {
return err
}

if containerNicName == "" {
return nil
}
Expand Down Expand Up @@ -835,7 +839,7 @@ func renameLink(curName, newName string) error {
return nil
}

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) (string, error) {
func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType, latency, limit, loss string, gwCheckMode int) (string, error) {
_, containerNicName := generateNicName(containerID, ifName)
ipStr := util.GetIpWithoutMask(ip)
ifaceID := ovs.PodNameToPortName(podName, podNamespace, provider)
Expand Down Expand Up @@ -863,6 +867,10 @@ func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace,
return containerNicName, err
}

if err = ovs.SetNetemQos(podName, podNamespace, ifaceID, latency, limit, loss); err != nil {
return containerNicName, err
}

podNS, err := ns.GetNS(netns)
if err != nil {
return containerNicName, fmt.Errorf("failed to open netns %q: %v", netns, err)
Expand Down
6 changes: 3 additions & 3 deletions pkg/daemon/ovs_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,11 @@ func (csh cniServerHandler) configureDpdkNic(podName, podNamespace, provider, ne
return errors.New("DPDK is not supported on Windows")
}

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) (string, error) {
return ifName, csh.configureNic(podName, podNamespace, provider, netns, containerID, "", ifName, mac, mtu, ip, gateway, isDefaultRoute, routes, dnsServer, dnsSuffix, ingress, egress, priority, DeviceID, nicType, gwCheckMode)
func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType, latency, limit, loss string, gwCheckMode int) (string, error) {
return ifName, csh.configureNic(podName, podNamespace, provider, netns, containerID, "", ifName, mac, mtu, ip, gateway, isDefaultRoute, routes, dnsServer, dnsSuffix, ingress, egress, priority, DeviceID, nicType, latency, limit, loss, gwCheckMode)
}

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType string, gwCheckMode int) error {
func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, isDefaultRoute bool, routes []request.Route, dnsServer, dnsSuffix []string, ingress, egress, priority, DeviceID, nicType, latency, limit, loss string, gwCheckMode int) error {
if DeviceID != "" {
return errors.New("SR-IOV is not supported on Windows")
}
Expand Down
59 changes: 59 additions & 0 deletions test/e2e/qos/qos.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,65 @@ var _ = Describe("[Qos]", func() {
Fail(err.Error())
}

It("create netem qos", func() {
name := f.GetName()
autoMount := false
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: map[string]string{
"e2e": "true",
"kubernetes.io/hostname": "kube-ovn-control-plane",
},
Annotations: map[string]string{
util.NetemQosLatencyAnnotation: "600",
util.NetemQosLimitAnnotation: "2000",
util.NetemQosLossAnnotation: "10",
},
},
Spec: corev1.PodSpec{
NodeName: "kube-ovn-control-plane",
Containers: []corev1.Container{
{
Name: name,
Image: testImage,
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
AutomountServiceAccountToken: &autoMount,
},
}

By("Create pod")
_, err := f.KubeClientSet.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())

_, err = f.WaitPodReady(name, namespace)
Expect(err).NotTo(HaveOccurred())

By("Check Qos annotation")
pod, err = f.KubeClientSet.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(pod.Annotations[util.AllocatedAnnotation]).To(Equal("true"))
Expect(pod.Annotations[util.RoutedAnnotation]).To(Equal("true"))
Expect(pod.Annotations[util.NetemQosLatencyAnnotation]).To(Equal("600"))
Expect(pod.Annotations[util.NetemQosLimitAnnotation]).To(Equal("2000"))
Expect(pod.Annotations[util.NetemQosLossAnnotation]).To(Equal("10"))

By("Check Ovs Qos Para")
time.Sleep(3 * time.Second)
qos, err := framework.GetPodNetemQosPara(name, namespace)
Expect(err).NotTo(HaveOccurred())
Expect(qos.Latency).To(Equal("600000"))
Expect(qos.Limit).To(Equal("2000"))
Expect(qos.Loss).To(Equal("10"))

By("Delete pod")
err = f.KubeClientSet.CoreV1().Pods(namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})

It("update netem qos", func() {
name := f.GetName()
autoMount := false
Expand Down

0 comments on commit 5160362

Please sign in to comment.