Skip to content

Commit

Permalink
add qos e2e test (#1505)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hongzhen-ma
hongzhen-ma committed May 7, 2022
1 parent f214ee2 commit 6942869
Show file tree
Hide file tree
Showing 4 changed files with 329 additions and 1 deletion.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ lint:
echo "Code differs from gofmt's style" 1>&2 && exit 1; \
fi
@GOOS=linux go vet ./...
@GOOS=linux gosec -exclude=G204,G601 -exclude-dir=test -exclude-dir=pkg/client ./...
@GOOS=linux gosec -exclude=G204,G601,G306 -exclude-dir=test -exclude-dir=pkg/client ./...

.PHONY: lint-windows
lint-windows:
Expand Down
1 change: 1 addition & 0 deletions test/e2e/e2e_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
_ "github.com/kubeovn/kube-ovn/test/e2e/ip"
_ "github.com/kubeovn/kube-ovn/test/e2e/kubectl-ko"
_ "github.com/kubeovn/kube-ovn/test/e2e/node"
_ "github.com/kubeovn/kube-ovn/test/e2e/qos"
_ "github.com/kubeovn/kube-ovn/test/e2e/service"
_ "github.com/kubeovn/kube-ovn/test/e2e/subnet"
"github.com/kubeovn/kube-ovn/test/e2e/underlay"
Expand Down
61 changes: 61 additions & 0 deletions test/e2e/framework/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
"fmt"
"io"
"os/exec"
"strings"
"time"

Expand Down Expand Up @@ -325,3 +326,63 @@ func getPodInitStatus(pod corev1.Pod, reason string) (bool, string) {
}
return initializing, reason
}

type netemQos struct {
Latency string `json:"latency"`
Limit string `json:"limit"`
Loss string `json:"loss"`
}

func GetPodNetemQosPara(podName, podNamespace string) (netemQos, error) {
var qosVal netemQos

output, err := exec.Command("kubectl", "ko", "vsctl", "kube-ovn-control-plane", "--no-heading", "--columns=other_config", "--bare", "find", "qos", fmt.Sprintf(`external_ids:pod="%s/%s"`, podNamespace, podName)).CombinedOutput()
if err != nil {
return qosVal, err
}

values := strings.Fields(string(output))
for _, val := range values {
temp := strings.Split(val, "=")
if len(temp) != 2 {
continue
}

switch temp[0] {
case "latency":
qosVal.Latency = temp[1]
case "limit":
qosVal.Limit = temp[1]
case "loss":
qosVal.Loss = temp[1]
}
}

return qosVal, nil
}

func GetPodHtbQosPara(podName, podNamespace string) (string, string, error) {
var priority, rate string

output, err := exec.Command("kubectl", "ko", "vsctl", "kube-ovn-control-plane", "--no-heading", "--columns=other_config", "--bare", "find", "queue", fmt.Sprintf(`external_ids:pod="%s/%s"`, podNamespace, podName)).CombinedOutput()
if err != nil {
return "", "", err
}

values := strings.Fields(string(output))
for _, val := range values {
temp := strings.Split(val, "=")
if len(temp) != 2 {
continue
}

switch temp[0] {
case "max-rate":
rate = temp[1]
case "priority":
priority = temp[1]
}
}

return priority, rate, nil
}
266 changes: 266 additions & 0 deletions test/e2e/qos/qos.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,266 @@
package qos

import (
"context"
"fmt"
"os"
"time"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"

. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"

kubeovn "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/util"
"github.com/kubeovn/kube-ovn/test/e2e/framework"
)

const testImage = "kubeovn/pause:3.2"

var _ = Describe("[Qos]", func() {
namespace := "qos"
f := framework.NewFramework("qos", fmt.Sprintf("%s/.kube/config", os.Getenv("HOME")))

ns := corev1.Namespace{
ObjectMeta: metav1.ObjectMeta{
Name: namespace,
Labels: map[string]string{"e2e": "true"},
},
}
if _, err := f.KubeClientSet.CoreV1().Namespaces().Create(context.Background(), &ns, metav1.CreateOptions{}); err != nil {
Fail(err.Error())
}

It("update netem qos", func() {
name := f.GetName()
autoMount := false
oriPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: map[string]string{
"e2e": "true",
"kubernetes.io/hostname": "kube-ovn-control-plane",
},
},
Spec: corev1.PodSpec{
NodeName: "kube-ovn-control-plane",
Containers: []corev1.Container{
{
Name: name,
Image: testImage,
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
AutomountServiceAccountToken: &autoMount,
},
}

By("Create pod")
_, err := f.KubeClientSet.CoreV1().Pods(namespace).Create(context.Background(), oriPod, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())

oriPod, err = f.WaitPodReady(name, namespace)
Expect(err).NotTo(HaveOccurred())
pod := oriPod.DeepCopy()

By("Annotate pod with netem qos")
pod.Annotations[util.NetemQosLatencyAnnotation] = "600"
pod.Annotations[util.NetemQosLimitAnnotation] = "2000"
pod.Annotations[util.NetemQosLossAnnotation] = "10"
patch, err := util.GenerateStrategicMergePatchPayload(oriPod, pod)
Expect(err).NotTo(HaveOccurred())

_, err = f.KubeClientSet.CoreV1().Pods(namespace).Patch(context.Background(), name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "")
Expect(err).NotTo(HaveOccurred())

pod, err = f.KubeClientSet.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(pod.Annotations[util.AllocatedAnnotation]).To(Equal("true"))
Expect(pod.Annotations[util.RoutedAnnotation]).To(Equal("true"))
Expect(pod.Annotations[util.NetemQosLatencyAnnotation]).To(Equal("600"))
Expect(pod.Annotations[util.NetemQosLimitAnnotation]).To(Equal("2000"))
Expect(pod.Annotations[util.NetemQosLossAnnotation]).To(Equal("10"))

By("Check ovs qos")
time.Sleep(3 * time.Second)
qos, err := framework.GetPodNetemQosPara(name, namespace)
Expect(err).NotTo(HaveOccurred())
Expect(qos.Latency).To(Equal("600000"))
Expect(qos.Limit).To(Equal("2000"))
Expect(qos.Loss).To(Equal("10"))

By("Delete pod")
err = f.KubeClientSet.CoreV1().Pods(namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})

It("create htb qos", func() {
name := f.GetName()
autoMount := false
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: map[string]string{
"e2e": "true",
"kubernetes.io/hostname": "kube-ovn-control-plane",
},
Annotations: map[string]string{
util.PriorityAnnotation: "50",
util.IngressRateAnnotation: "300",
},
},
Spec: corev1.PodSpec{
NodeName: "kube-ovn-control-plane",
Containers: []corev1.Container{
{
Name: name,
Image: testImage,
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
AutomountServiceAccountToken: &autoMount,
},
}

By("Create pod")
_, err := f.KubeClientSet.CoreV1().Pods(namespace).Create(context.Background(), pod, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())

_, err = f.WaitPodReady(name, namespace)
Expect(err).NotTo(HaveOccurred())

By("Check Qos annotation")
pod, err = f.KubeClientSet.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(pod.Annotations[util.AllocatedAnnotation]).To(Equal("true"))
Expect(pod.Annotations[util.RoutedAnnotation]).To(Equal("true"))
Expect(pod.Annotations[util.PriorityAnnotation]).To(Equal("50"))
Expect(pod.Annotations[util.IngressRateAnnotation]).To(Equal("300"))

By("Check Ovs Qos Para")
time.Sleep(3 * time.Second)
priority, rate, err := framework.GetPodHtbQosPara(name, namespace)
Expect(err).NotTo(HaveOccurred())
Expect(priority).To(Equal("50"))
Expect(rate).To(Equal("300000000"))

By("Delete pod")
err = f.KubeClientSet.CoreV1().Pods(namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})

It("update htb qos", func() {
name := f.GetName()
cidr := "20.6.0.0/16"

By("create subnet with htbqos")
s := kubeovn.Subnet{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Labels: map[string]string{"e2e": "true"},
},
Spec: kubeovn.SubnetSpec{
CIDRBlock: cidr,
Protocol: util.CheckProtocol(cidr),
HtbQos: util.HtbQosHigh,
Namespaces: []string{namespace},
},
}
_, err := f.OvnClientSet.KubeovnV1().Subnets().Create(context.Background(), &s, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())

err = f.WaitSubnetReady(name)
Expect(err).NotTo(HaveOccurred())

subnet, err := f.OvnClientSet.KubeovnV1().Subnets().Get(context.Background(), name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(subnet.Spec.HtbQos).To(Equal(util.HtbQosHigh))

autoMount := false
oriPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
Labels: map[string]string{
"e2e": "true",
"kubernetes.io/hostname": "kube-ovn-control-plane",
},
Annotations: map[string]string{
util.LogicalSwitchAnnotation: subnet.Name,
},
},
Spec: corev1.PodSpec{
NodeName: "kube-ovn-control-plane",
Containers: []corev1.Container{
{
Name: name,
Image: testImage,
ImagePullPolicy: corev1.PullIfNotPresent,
},
},
AutomountServiceAccountToken: &autoMount,
},
}

By("Create pod")
_, err = f.KubeClientSet.CoreV1().Pods(namespace).Create(context.Background(), oriPod, metav1.CreateOptions{})
Expect(err).NotTo(HaveOccurred())

oriPod, err = f.WaitPodReady(name, namespace)
Expect(err).NotTo(HaveOccurred())

By("Check Ovs Qos Para, same as subnet")
time.Sleep(10 * time.Second)
priority, _, err := framework.GetPodHtbQosPara(name, namespace)
Expect(err).NotTo(HaveOccurred())
Expect(priority).To(Equal("100"))

By("Annotate pod with priority")
pod := oriPod.DeepCopy()
pod.Annotations[util.PriorityAnnotation] = "60"

patch, err := util.GenerateStrategicMergePatchPayload(oriPod, pod)
Expect(err).NotTo(HaveOccurred())

_, err = f.KubeClientSet.CoreV1().Pods(namespace).Patch(context.Background(), name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "")
Expect(err).NotTo(HaveOccurred())

pod, err = f.KubeClientSet.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())
Expect(pod.Annotations[util.PriorityAnnotation]).To(Equal("60"))

By("Check Ovs Qos Para")
time.Sleep(3 * time.Second)
priority, _, err = framework.GetPodHtbQosPara(name, namespace)
Expect(err).NotTo(HaveOccurred())
Expect(priority).To(Equal("60"))

By("Delete Pod priority annotation")
testPod := pod.DeepCopy()
delete(testPod.Annotations, util.PriorityAnnotation)

patch, err = util.GenerateStrategicMergePatchPayload(pod, testPod)
Expect(err).NotTo(HaveOccurred())

_, err = f.KubeClientSet.CoreV1().Pods(namespace).Patch(context.Background(), name, types.StrategicMergePatchType, patch, metav1.PatchOptions{}, "")
Expect(err).NotTo(HaveOccurred())

_, err = f.KubeClientSet.CoreV1().Pods(namespace).Get(context.Background(), name, metav1.GetOptions{})
Expect(err).NotTo(HaveOccurred())

By("Check Ovs Qos Para, priority from subnet")
time.Sleep(10 * time.Second)
priority, _, err = framework.GetPodHtbQosPara(name, namespace)
Expect(err).NotTo(HaveOccurred())
Expect(priority).To(Equal("100"))

By("Delete pod")
err = f.KubeClientSet.CoreV1().Pods(namespace).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
Expect(err).NotTo(HaveOccurred())
})
})

0 comments on commit 6942869

Please sign in to comment.