optimize ovs request in cni (#1518)

kubeovn · May 13, 2022 · 4eebabc · 4eebabc
1 parent 7a3f73d
commit 4eebabc
Show file tree

Hide file tree

Showing 7 changed files with 60 additions and 40 deletions.
diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go
@@ -628,6 +628,22 @@ func (c *Controller) deleteSubnetQos(subnet *kubeovnv1.Subnet) error {
 	return nil
 }
 
+func (c *Controller) getSubnetQosPriority(subnetName string) string {
+	var priority string
+	subnet, err := c.subnetsLister.Get(subnetName)
+	if err != nil {
+		klog.Errorf("failed to get subnet %s: %v", subnet, err)
+	} else if subnet.Spec.HtbQos != "" {
+		htbQos, err := c.htbQosLister.Get(subnet.Spec.HtbQos)
+		if err != nil {
+			klog.Errorf("failed to get htbqos %s: %v", subnet.Spec.HtbQos, err)
+		} else {
+			priority = htbQos.Spec.Priority
+		}
+	}
+	return priority
+}
+
 // Run starts controller
 func (c *Controller) Run(stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
@@ -664,7 +680,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) {
 			klog.Errorf("gc ovs port error: %v", err)
 		}
 	}, 5*time.Minute, stopCh)
-	go wait.Until(c.loopCheckSubnetQosPriority, 5*time.Second, stopCh)
+
 	<-stopCh
 	klog.Info("Shutting down workers")
 }
diff --git a/pkg/daemon/controller_linux.go b/pkg/daemon/controller_linux.go
@@ -176,6 +176,18 @@ func (c *Controller) reconcileRouters(event subnetEvent) error {
 			}
 		}
 	}
+
+	if oldSubnet != nil && newSubnet != nil && oldSubnet.Spec.HtbQos != "" && newSubnet.Spec.HtbQos == "" {
+		if err := c.deleteSubnetQos(newSubnet); err != nil {
+			klog.Errorf("failed to delete htb qos for subnet %s: %v", newSubnet.Name, err)
+			return err
+		}
+	} else if newSubnet != nil && newSubnet.Spec.HtbQos != "" {
+		if err := c.setSubnetQosPriority(newSubnet); err != nil {
+			klog.Errorf("failed to set htb qos priority for subnet %s: %v", newSubnet.Name, err)
+			return err
+		}
+	}
 	return nil
 }
 
@@ -403,9 +415,16 @@ func (c *Controller) handlePod(key string) error {
 		podName = pod.Annotations[fmt.Sprintf(util.VmTemplate, util.OvnProvider)]
 	}
 
+	priority := pod.Annotations[util.PriorityAnnotation]
+	subnetName := pod.Annotations[util.LogicalSwitchAnnotation]
+	subnetPriority := c.getSubnetQosPriority(subnetName)
+	if priority == "" && subnetPriority != "" {
+		priority = subnetPriority
+	}
+
 	// set default nic bandwidth
 	ifaceID := ovs.PodNameToPortName(podName, pod.Namespace, util.OvnProvider)
-	err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[util.EgressRateAnnotation], pod.Annotations[util.IngressRateAnnotation], pod.Annotations[util.PriorityAnnotation])
+	err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[util.EgressRateAnnotation], pod.Annotations[util.IngressRateAnnotation], priority)
 	if err != nil {
 		return err
 	}
@@ -431,7 +450,14 @@ func (c *Controller) handlePod(key string) error {
 		}
 		if pod.Annotations[fmt.Sprintf(util.AllocatedAnnotationTemplate, provider)] == "true" {
 			ifaceID = ovs.PodNameToPortName(podName, pod.Namespace, provider)
-			err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[fmt.Sprintf(util.EgressRateAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.IngressRateAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.PriorityAnnotationTemplate, provider)])
+			priority := pod.Annotations[fmt.Sprintf(util.PriorityAnnotationTemplate, provider)]
+			subnetName := pod.Annotations[fmt.Sprintf(util.LogicalSwitchAnnotationTemplate, provider)]
+			subnetPriority := c.getSubnetQosPriority(subnetName)
+			if priority == "" && subnetPriority != "" {
+				priority = subnetPriority
+			}
+
+			err = ovs.SetInterfaceBandwidth(podName, pod.Namespace, ifaceID, pod.Annotations[fmt.Sprintf(util.EgressRateAnnotationTemplate, provider)], pod.Annotations[fmt.Sprintf(util.IngressRateAnnotationTemplate, provider)], priority)
 			if err != nil {
 				return err
 			}
@@ -491,28 +517,6 @@ func (c *Controller) loopEncapIpCheck() {
 	}
 }
 
-func (c *Controller) loopCheckSubnetQosPriority() {
-	subnets, err := c.subnetsLister.List(labels.Everything())
-	if err != nil {
-		klog.Errorf("failed to list subnets %v", err)
-		return
-	}
-
-	for _, subnet := range subnets {
-		if subnet.Spec.HtbQos == "" {
-			if err := c.deleteSubnetQos(subnet); err != nil {
-				klog.Errorf("failed to delete htb qos for subnet %s: %v", subnet.Name, err)
-				return
-			}
-		} else {
-			if err := c.setSubnetQosPriority(subnet); err != nil {
-				klog.Errorf("failed to set htb qos priority for subnet %s: %v", subnet.Name, err)
-				return
-			}
-		}
-	}
-}
-
 func (c *Controller) setSubnetQosPriority(subnet *kubeovnv1.Subnet) error {
 	htbQos, err := c.htbQosLister.Get(subnet.Spec.HtbQos)
 	if err != nil {

diff --git a/pkg/daemon/controller_windows.go b/pkg/daemon/controller_windows.go
@@ -228,10 +228,6 @@ func (c *Controller) loopEncapIpCheck() {
 	// TODO
 }
 
-func (c *Controller) loopCheckSubnetQosPriority() {
-	// TODO
-}
-
 func (c *Controller) clearQos(podName, podNamespace, ifaceID string) error {
 	// TODO
 	return nil

diff --git a/pkg/daemon/gateway.go b/pkg/daemon/gateway.go
@@ -47,6 +47,11 @@ func (c *Controller) setGatewayBandwidth() error {
 	}
 	ingress, egress, priority := node.Annotations[util.IngressRateAnnotation], node.Annotations[util.EgressRateAnnotation], node.Annotations[util.PriorityAnnotation]
 	ifaceId := fmt.Sprintf("node-%s", c.config.NodeName)
+	if ingress == "" && egress == "" && priority == "" {
+		if htbQos, _ := ovs.IsHtbQos(ifaceId); !htbQos {
+			return nil
+		}
+	}
 	return ovs.SetInterfaceBandwidth("", "", ifaceId, egress, ingress, priority)
 }
 

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -203,6 +203,11 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			return
 		}
 
+		subnetPriority := csh.Controller.getSubnetQosPriority(subnet)
+		if priority == "" && subnetPriority != "" {
+			priority = subnetPriority
+		}
+
 		//skip ping check gateway for pods during live migration
 		if pod.Annotations[fmt.Sprintf(util.LiveMigrationAnnotationTemplate, podRequest.Provider)] != "true" {
 			if !podSubnet.Spec.DisableGatewayCheck {

diff --git a/pkg/ovs/ovs-vsctl_linux.go b/pkg/ovs/ovs-vsctl_linux.go
@@ -117,19 +117,13 @@ func ClearHtbQosQueue(podName, podNamespace, iface string) error {
 }
 
 func IsHtbQos(iface string) (bool, error) {
-	qosList, err := ovsFind("qos", "_uuid", fmt.Sprintf(`external-ids:iface-id="%s"`, iface))
+	qosType, err := ovsFind("qos", "type", fmt.Sprintf(`external-ids:iface-id="%s"`, iface))
 	if err != nil {
 		return false, err
 	}
 
-	for _, qos := range qosList {
-		qosType, err := ovsGet("qos", qos, "type", "")
-		if err != nil {
-			return false, err
-		}
-		if qosType == util.HtbQos {
-			return true, nil
-		}
+	if len(qosType) != 0 && qosType[0] == util.HtbQos {
+		return true, nil
 	}
 	return false, nil
 }

diff --git a/test/e2e/qos/qos.go b/test/e2e/qos/qos.go
@@ -279,7 +279,7 @@ var _ = Describe("[Qos]", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Check Ovs Qos Para, same as subnet")
-		time.Sleep(10 * time.Second)
+		time.Sleep(3 * time.Second)
 		priority, _, err := framework.GetPodHtbQosPara(name, namespace)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(priority).To(Equal("100"))
@@ -318,7 +318,7 @@ var _ = Describe("[Qos]", func() {
 		Expect(err).NotTo(HaveOccurred())
 
 		By("Check Ovs Qos Para, priority from subnet")
-		time.Sleep(10 * time.Second)
+		time.Sleep(3 * time.Second)
 		priority, _, err = framework.GetPodHtbQosPara(name, namespace)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(priority).To(Equal("100"))