e2e: add qos policy test cases (#2924)

* e2e: add qos policy test cases

Set qos policy for natgw
Set qos policy for eip
Rebuild qos of natgw when the natgw pod restarts
Rebuild qos of eip when the natgw pod restarts
Change qos policy of natgw
Change qos policy of eip
Update qos policy of eip
Set specific ip qos policy of natgw
Match Qos priority
Create natgw with qos policy
Create eip with qos policy

* Make e2e run faster

pkg/controller/qos_policy.go

@@ -400,7 +400,8 @@ func (c *Controller) handleUpdateQoSPolicy(key string) error {
 		if cachedQos.Spec.BindingType == kubeovnv1.QoSBindingTypeEIP {
 			eips, err := c.iptablesEipsLister.List(
 				labels.SelectorFromSet(labels.Set{util.QoSLabel: key}))
-			if err != nil {
+			// when eip is not found, we should delete finalizer
+			if err != nil && !k8serrors.IsNotFound(err) {
 				klog.Errorf("failed to get eip list, %v", err)
 				return err
 			}
@@ -414,7 +415,8 @@ func (c *Controller) handleUpdateQoSPolicy(key string) error {
 		if cachedQos.Spec.BindingType == kubeovnv1.QoSBindingTypeNatGw {
 			gws, err := c.vpcNatGatewayLister.List(
 				labels.SelectorFromSet(labels.Set{util.QoSLabel: key}))
-			if err != nil {
+			// when nat gw is not found, we should delete finalizer
+			if err != nil && !k8serrors.IsNotFound(err) {
 				klog.Errorf("failed to get gw list, %v", err)
 				return err
 			}

test/e2e/framework/docker/network.go

@@ -139,3 +139,12 @@ func NetworkDisconnect(networkID, containerID string) error {
 
 	return cli.NetworkDisconnect(context.Background(), networkID, containerID, false)
 }
+
+func NetworkRemove(networkID string) error {
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	if err != nil {
+		return err
+	}
+	defer cli.Close()
+	return cli.NetworkRemove(context.Background(), networkID)
+}

test/e2e/framework/exec_utils.go

@@ -0,0 +1,34 @@
+package framework
+
+import (
+	"context"
+
+	"github.com/kubeovn/kube-ovn/pkg/util"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/kubernetes/test/e2e/framework"
+
+	"github.com/onsi/gomega"
+)
+
+// ExecCommandInContainer executes a command in the specified container.
+func ExecCommandInContainer(f *Framework, podName, containerName string, cmd ...string) (string, string, error) {
+	return util.ExecuteCommandInContainer(f.ClientSet, f.ClientConfig(), f.Namespace.Name, podName, containerName, cmd...)
+}
+
+// ExecShellInContainer executes the specified command on the pod's container.
+func ExecShellInContainer(f *Framework, podName, containerName string, cmd string) (string, string, error) {
+	return ExecCommandInContainer(f, podName, containerName, "/bin/sh", "-c", cmd)
+}
+
+func execCommandInPod(ctx context.Context, f *Framework, podName string, cmd ...string) (string, string, error) {
+	pod, err := f.PodClient().Get(ctx, podName, metav1.GetOptions{})
+	framework.ExpectNoError(err, "failed to get pod %v", podName)
+	gomega.Expect(pod.Spec.Containers).NotTo(gomega.BeEmpty())
+	return ExecCommandInContainer(f, podName, pod.Spec.Containers[0].Name, cmd...)
+}
+
+// ExecShellInPod executes the specified command on the pod.
+func ExecShellInPod(ctx context.Context, f *Framework, podName string, cmd string) (string, string, error) {
+	return execCommandInPod(ctx, f, podName, "/bin/sh", "-c", cmd)
+}

test/e2e/framework/iptables-eip.go

@@ -90,6 +90,17 @@ func (c *IptablesEIPClient) PatchSync(original, modified *apiv1.IptablesEIP, req
 	return c.Get(eip.Name).DeepCopy()
 }
 
+// PatchQoS patches the vpc nat gw and waits for the qos to be ready for `timeout`.
+// If the qos doesn't become ready before the timeout, it will fail the test.
+func (c *IptablesEIPClient) PatchQoSPolicySync(eipName string, qosPolicyName string) *apiv1.IptablesEIP {
+	eip := c.Get(eipName)
+	modifiedEIP := eip.DeepCopy()
+	modifiedEIP.Spec.QoSPolicy = qosPolicyName
+	_ = c.Patch(eip, modifiedEIP)
+	ExpectTrue(c.WaitToQoSReady(eipName))
+	return c.Get(eipName).DeepCopy()
+}
+
 // Delete deletes a iptables eip if the iptables eip exists
 func (c *IptablesEIPClient) Delete(name string) {
 	err := c.IptablesEIPInterface.Delete(context.TODO(), name, metav1.DeleteOptions{})
@@ -118,6 +129,19 @@ func (c *IptablesEIPClient) WaitToBeReady(name string, timeout time.Duration) bo
 	return false
 }
 
+// WaitToQoSReady returns whether the qos is ready within timeout.
+func (c *IptablesEIPClient) WaitToQoSReady(name string) bool {
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(poll) {
+		eip := c.Get(name)
+		if eip.Status.QoSPolicy == eip.Spec.QoSPolicy {
+			Logf("qos %s is ready ", name)
+			return true
+		}
+		Logf("qos %s is not ready ", name)
+	}
+	return false
+}
+
 // WaitToBeUpdated returns whether the iptables eip is updated within timeout.
 func (c *IptablesEIPClient) WaitToBeUpdated(eip *apiv1.IptablesEIP, timeout time.Duration) bool {
 	Logf("Waiting up to %v for iptables eip %s to be updated", timeout, eip.Name)
@@ -147,7 +171,7 @@ func (c *IptablesEIPClient) WaitToDisappear(name string, interval, timeout time.
 	return nil
 }
 
-func MakeIptablesEIP(name, v4ip, v6ip, mac, natGwDp, externalSubnet string) *apiv1.IptablesEIP {
+func MakeIptablesEIP(name, v4ip, v6ip, mac, natGwDp, externalSubnet, qosPolicyName string) *apiv1.IptablesEIP {
 	eip := &apiv1.IptablesEIP{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: name,
@@ -162,5 +186,6 @@ func MakeIptablesEIP(name, v4ip, v6ip, mac, natGwDp, externalSubnet string) *api
 	if externalSubnet != "" {
 		eip.Spec.ExternalSubnet = externalSubnet
 	}
+	eip.Spec.QoSPolicy = qosPolicyName
 	return eip
 }