fix: nat rules can be modified

(cherry picked from commit 4d8b186)

docs/snat-and-eip.md

@@ -85,6 +85,13 @@ spec:
     image: nginx:alpine
 ```
 
+4. Change eip or snat ip
+```bash
+# ovn.kubernetes.io/routed annotation need to be removed to trigger control plan update
+kubectl annotate pod pod-gw ovn.kubernetes.io/eip=172.56.0.221 --overwrite
+kubectl annotate pod pod-gw ovn.kubernetes.io/routed-
+```
+
 ## Limitations
 * No IP conflict detection for now, users should control the nat address allocation by themselves.
 

pkg/controller/pod.go

@@ -476,7 +476,7 @@ func (c *Controller) handleUpdatePod(key string) error {
 		return err
 	}
 	if !subnet.Spec.UnderlayGateway {
-		if pod.Annotations[util.EipAnnotation] != "" {
+		if pod.Annotations[util.EipAnnotation] != "" || pod.Annotations[util.SnatAnnotation] != "" {
 			cm, err := c.configMapsLister.ConfigMaps("kube-system").Get(util.ExternalGatewayConfig)
 			if err != nil {
 				klog.Errorf("failed to get ex-gateway config, %v", err)
@@ -493,40 +493,8 @@ func (c *Controller) handleUpdatePod(key string) error {
 				klog.Errorf("failed to add static route, %v", err)
 				return err
 			}
-
-			if err := c.ovnClient.AddNatRule("dnat_and_snat", podIP, pod.Annotations[util.EipAnnotation], c.config.ClusterRouter); err != nil {
-				klog.Errorf("failed to add nat rules, %v", err)
-				return err
-			}
-		} else if pod.Annotations[util.SnatAnnotation] != "" {
-			cm, err := c.configMapsLister.ConfigMaps("kube-system").Get(util.ExternalGatewayConfig)
-			if err != nil {
-				klog.Errorf("failed to get ex-gateway config, %v", err)
-				return err
-			}
-			nextHop := cm.Data["nic-ip"]
-			if nextHop == "" {
-				klog.Errorf("no available gateway nic address")
-				return fmt.Errorf("no available gateway nic address")
-			}
-			nextHop = strings.Split(nextHop, "/")[0]
-
-			if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, podIP, nextHop, c.config.ClusterRouter); err != nil {
-				klog.Errorf("failed to add static route, %v", err)
-				return err
-			}
-
-			if err := c.ovnClient.AddNatRule("snat", podIP, pod.Annotations[util.SnatAnnotation], c.config.ClusterRouter); err != nil {
-				klog.Errorf("failed to add nat rules, %v", err)
-				return err
-			}
-		} else if pod.Annotations[util.NorthGatewayAnnotation] != "" {
-			if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation], c.config.ClusterRouter); err != nil {
-				klog.Errorf("failed to add static route, %v", err)
-				return err
-			}
 		} else {
-			if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType {
+			if subnet.Spec.GatewayType == kubeovnv1.GWDistributedType && pod.Annotations[util.NorthGatewayAnnotation] == "" {
 				node, err := c.nodesLister.Get(pod.Spec.NodeName)
 				if err != nil {
 					klog.Errorf("get node %s failed %v", pod.Spec.NodeName, err)
@@ -542,6 +510,23 @@ func (c *Controller) handleUpdatePod(key string) error {
 					return err
 				}
 			}
+
+			if pod.Annotations[util.NorthGatewayAnnotation] != "" {
+				if err := c.ovnClient.AddStaticRoute(ovs.PolicySrcIP, podIP, pod.Annotations[util.NorthGatewayAnnotation], c.config.ClusterRouter); err != nil {
+					klog.Errorf("failed to add static route, %v", err)
+					return err
+				}
+			}
+		}
+
+		if err := c.ovnClient.UpdateNatRule("dnat_and_snat", podIP, pod.Annotations[util.EipAnnotation], c.config.ClusterRouter); err != nil {
+			klog.Errorf("failed to add nat rules, %v", err)
+			return err
+		}
+
+		if err := c.ovnClient.UpdateNatRule("snat", podIP, pod.Annotations[util.SnatAnnotation], c.config.ClusterRouter); err != nil {
+			klog.Errorf("failed to add nat rules, %v", err)
+			return err
 		}
 	}
 

pkg/ovs/ovn-nbctl.go

@@ -409,9 +409,38 @@ func (c Client) AddStaticRoute(policy, cidr, nextHop, router string) error {
 	return err
 }
 
-func (c Client) AddNatRule(policy, logicalIP, externalIP, router string) error {
-	_, err := c.ovnNbCommand(MayExist, "lr-nat-add", router, policy, externalIP, logicalIP)
-	return err
+func (c Client) UpdateNatRule(policy, logicalIP, externalIP, router string) error {
+	if policy == "snat" {
+		if externalIP == "" {
+			_, err := c.ovnNbCommand(IfExists, "lr-nat-del", router, "snat", logicalIP)
+			return err
+		}
+		_, err := c.ovnNbCommand(IfExists, "lr-nat-del", router, "snat", logicalIP, "--",
+			MayExist, "lr-nat-add", router, policy, externalIP, logicalIP)
+		return err
+	} else {
+		output, err := c.ovnNbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=external_ip", "find", "NAT", fmt.Sprintf("logical_ip=%s", logicalIP), "type=dnat_and_snat")
+		if err != nil {
+			klog.Errorf("failed to list nat rules, %v", err)
+			return err
+		}
+		eips := strings.Split(output, "\n")
+		for _, eip := range eips {
+			eip = strings.TrimSpace(eip)
+			if eip == "" || eip == externalIP {
+				continue
+			}
+			if _, err := c.ovnNbCommand(IfExists, "lr-nat-del", router, "dnat_and_snat", eip); err != nil {
+				klog.Errorf("failed to delete nat rule, %v", err)
+				return err
+			}
+		}
+		if externalIP != "" {
+			_, err = c.ovnNbCommand(MayExist, "lr-nat-add", router, policy, externalIP, logicalIP)
+			return err
+		}
+	}
+	return nil
 }
 
 func (c Client) DeleteNatRule(logicalIP, router string) error {

pkg/ovs/ovn-sbctl.go

@@ -2,6 +2,7 @@ package ovs
 
 import (
 	"fmt"
+	"os"
 	"os/exec"
 	"strings"
 	"time"
@@ -11,7 +12,18 @@ import (
 
 func (c Client) ovnSbCommand(cmdArgs ...string) (string, error) {
 	start := time.Now()
-	cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnTimeout), fmt.Sprintf("--db=%s", c.OvnSbAddress)}, cmdArgs...)
+	if os.Getenv("ENABLE_SSL") == "true" {
+		cmdArgs = append([]string{
+			fmt.Sprintf("--timeout=%d", c.OvnTimeout),
+			fmt.Sprintf("--db=%s", c.OvnSbAddress),
+			"-p", "/var/run/tls/key",
+			"-c", "/var/run/tls/cert",
+			"-C", "/var/run/tls/cacert"}, cmdArgs...)
+	} else {
+		cmdArgs = append([]string{
+			fmt.Sprintf("--timeout=%d", c.OvnTimeout),
+			fmt.Sprintf("--db=%s", c.OvnSbAddress)}, cmdArgs...)
+	}
 	raw, err := exec.Command(OvnSbCtl, cmdArgs...).CombinedOutput()
 	elapsed := float64((time.Since(start)) / time.Millisecond)
 	klog.V(4).Infof("%s command %s in %vms", OvnSbCtl, strings.Join(cmdArgs, " "), elapsed)