support node local dns cache (#2733)

* support node local dns cache

* remove Nodelocaldnsip in kube-ovn-cni

* when remove --node-local-dns-ip ,should remove related policy route

* fix change node local dns cache ip change

charts/templates/controller-deploy.yaml

@@ -106,6 +106,7 @@ spec:
           - --keep-vm-ip={{- .Values.func.ENABLE_KEEP_VM_IP }}
           - --pod-default-fip-type={{- .Values.networking.POD_DEFAULT_FIP_TYPE }}
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
+          - --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }}
           env:
             - name: ENABLE_SSL
               value: "{{ .Values.networking.ENABLE_SSL }}"

charts/templates/ovncni-ds.yaml

@@ -52,7 +52,6 @@ spec:
           - --enable-mirror={{- .Values.debug.ENABLE_MIRROR }}
           - --mirror-iface={{- .Values.debug.MIRROR_IFACE }}
           - --node-switch={{ .Values.networking.NODE_SUBNET }}
-          - --node-local-dns-ip={{ .Values.networking.NODE_LOCAL_DNS_IP }}
           - --encap-checksum=true
           - --service-cluster-ip-range=
           {{- if eq .Values.networking.NET_STACK "dual_stack" -}}

dist/images/install.sh

@@ -20,6 +20,7 @@ CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01}
 ENABLE_LB_SVC=${ENABLE_LB_SVC:-false}
 ENABLE_NAT_GW=${ENABLE_NAT_GW:-false}
 ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true}
+NODE_LOCAL_DNS_IP=${NODE_LOCAL_DNS_IP:-}
 # exchange link names of OVS bridge and the provider nic
 # in the default provider-network
 EXCHANGE_LINK_NAME=${EXCHANGE_LINK_NAME:-false}
@@ -3602,6 +3603,7 @@ spec:
           - --enable-lb-svc=$ENABLE_LB_SVC
           - --keep-vm-ip=$ENABLE_KEEP_VM_IP
           - --pod-default-fip-type=$POD_DEFAULT_FIP_TYPE
+          - --node-local-dns-ip=$NODE_LOCAL_DNS_IP
           env:
             - name: ENABLE_SSL
               value: "$ENABLE_SSL"

pkg/controller/config.go

@@ -99,6 +99,8 @@ type Configuration struct {
 	BfdMinTx      int
 	BfdMinRx      int
 	BfdDetectMult int
+
+	NodeLocalDnsIP string
 }
 
 // ParseFlags parses cmd args then init kubeclient and conf
@@ -162,6 +164,7 @@ func ParseFlags() (*Configuration, error) {
 		argExternalGatewaySwitch   = pflag.String("external-gateway-switch", "external", "The name of the external gateway switch which is a ovs bridge to provide external network, default: external")
 		argExternalGatewayNet      = pflag.String("external-gateway-net", "external", "The name of the external network which mappings with an ovs bridge, default: external")
 		argExternalGatewayVlanID   = pflag.Int("external-gateway-vlanid", 0, "The vlanId of port ln-ovn-external, default: 0")
+		argNodeLocalDnsIP          = pflag.String("node-local-dns-ip", "", "The node local dns ip , this feature is using the local dns cache in k8s")
 
 		argGCInterval      = pflag.Int("gc-interval", 360, "The interval between GC processes, default 360 seconds")
 		argInspectInterval = pflag.Int("inspect-interval", 20, "The interval between inspect processes, default 20 seconds")
@@ -246,6 +249,7 @@ func ParseFlags() (*Configuration, error) {
 		BfdMinTx:                       *argBfdMinTx,
 		BfdMinRx:                       *argBfdMinRx,
 		BfdDetectMult:                  *argBfdDetectMult,
+		NodeLocalDnsIP:                 *argNodeLocalDnsIP,
 	}
 
 	if config.NetworkType == util.NetworkTypeVlan && config.DefaultHostInterface == "" {

pkg/controller/node.go

@@ -273,11 +273,21 @@ func (c *Controller) handleAddNode(key string) error {
 				"node":           node.Name,
 				"address-family": strconv.Itoa(af),
 			}
-			klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, extrenalID %v", c.config.ClusterRouter, match, "reroute", ip, externalIDs)
+			klog.Infof("add policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, "reroute", ip, externalIDs)
 			if err = c.ovnLegacyClient.AddPolicyRoute(c.config.ClusterRouter, util.NodeRouterPolicyPriority, match, "reroute", ip, externalIDs); err != nil {
 				klog.Errorf("failed to add logical router policy for node %s: %v", node.Name, err)
 				return err
 			}
+
+			if err = c.deletePolicyRouteForLocalDnsCacheOnNode(portName, node.Name, af); err != nil {
+				return err
+			}
+
+			if c.config.NodeLocalDnsIP != "" {
+				if err = c.addPolicyRouteForLocalDnsCacheOnNode(portName, ip, node.Name, af); err != nil {
+					return err
+				}
+			}
 		}
 	}
 
@@ -459,6 +469,13 @@ func (c *Controller) handleDeleteNode(key string) error {
 		return err
 	}
 
+	afs := []int{4, 6}
+	for _, af := range afs {
+		if err := c.deletePolicyRouteForLocalDnsCacheOnNode(portName, key, af); err != nil {
+			return err
+		}
+	}
+
 	// ovn acl doesn't support address_set name with '-', so replace '-' by '.'
 	pgName := strings.Replace(portName, "-", ".", -1)
 	if err := c.ovnClient.DeletePortGroup(pgName); err != nil {
@@ -1211,3 +1228,50 @@ func (c *Controller) addPolicyRouteForCentralizedSubnetOnNode(nodeName, nodeIP s
 	}
 	return nil
 }
+
+func (c *Controller) addPolicyRouteForLocalDnsCacheOnNode(nodePortName, nodeIP, nodeName string, af int) error {
+	externalIDs := map[string]string{
+		"vendor":          util.CniTypeName,
+		"node":            nodeName,
+		"address-family":  strconv.Itoa(af),
+		"isLocalDnsCache": "true",
+	}
+
+	pgAs := strings.Replace(fmt.Sprintf("%s_ip%d", nodePortName, af), "-", ".", -1)
+	match := fmt.Sprintf("ip%d.src == $%s && ip%d.dst == %s", af, pgAs, af, c.config.NodeLocalDnsIP)
+	klog.Infof("add node local dns cache policy route for router: %s, match %s, action %s, nexthop %s, externalID %v", c.config.ClusterRouter, match, "reroute", nodeIP, externalIDs)
+	if err := c.ovnLegacyClient.AddPolicyRoute(c.config.ClusterRouter, util.NodeLocalDnsPolicyPriority, match, "reroute", nodeIP, externalIDs); err != nil {
+		klog.Errorf("failed to add logical router policy for node %s: %v", nodeName, err)
+		return err
+	}
+	return nil
+}
+
+func (c *Controller) deletePolicyRouteForLocalDnsCacheOnNode(nodePortName, nodeName string, af int) error {
+	results, err := c.ovnLegacyClient.CustomFindEntity("Logical_Router_Policy", []string{"_uuid", "match", "priority"},
+		fmt.Sprintf("external_ids:vendor=\"%s\"", util.CniTypeName),
+		fmt.Sprintf("external_ids:node=\"%s\"", nodeName),
+		fmt.Sprintf("external_ids:address-family=\"%s\"", strconv.Itoa(af)),
+		"external_ids:isLocalDnsCache=\"true\"",
+	)
+	if err != nil {
+		klog.Errorf("customFindEntity failed, %v", err)
+		return err
+	}
+
+	if len(results) == 0 {
+		return nil
+	}
+
+	var uuids []string
+	for _, result := range results {
+		uuids = append(uuids, result["_uuid"][0])
+		klog.Infof("delete node local dns cache policy route for router %s with match %s ", c.config.ClusterRouter, result["match"])
+
+		if err := c.ovnLegacyClient.DeletePolicyRouteByUUID(c.config.ClusterRouter, uuids); err != nil {
+			klog.Errorf("failed to delete policy route for node local dns in router %s with match %s : %v", c.config.ClusterRouter, result["match"], err)
+			return err
+		}
+	}
+	return nil
+}

pkg/controller/pod.go

@@ -611,7 +611,7 @@ func (c *Controller) reconcileAllocateSubnets(cachedPod, pod *v1.Pod, needAlloca
 		}
 
 		if podNet.Type != providerTypeIPAM {
-			if (subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) && subnet.Spec.Vpc != "" {
+			if (subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway || subnet.Spec.U2OInterconnection) && subnet.Spec.Vpc != "" {
 				pod.Annotations[fmt.Sprintf(util.LogicalRouterAnnotationTemplate, podNet.ProviderName)] = subnet.Spec.Vpc
 			}
 

pkg/controller/subnet.go

@@ -1205,7 +1205,7 @@ func (c *Controller) reconcileVpcAddNormalStaticRoute(vpcName string) error {
 
 	defualtExternalSubnet, err := c.subnetsLister.Get(c.config.ExternalGatewaySwitch)
 	if err != nil {
-		klog.Error("failed to get default external switch subnet %s: %v", c.config.ExternalGatewaySwitch)
+		klog.Error("failed to get default external switch subnet %s: %v", c.config.ExternalGatewaySwitch, err)
 		return err
 	}
 	gatewayV4, gatewayV6 := util.SplitStringIP(defualtExternalSubnet.Spec.Gateway)

pkg/daemon/config.go

@@ -44,7 +44,6 @@ type Configuration struct {
 	NodeName                string
 	ServiceClusterIPRange   string
 	NodeSwitch              string
-	NodeLocalDnsIP          string
 	EncapChecksum           bool
 	EnablePprof             bool
 	MacLearningFallback     bool
@@ -75,7 +74,6 @@ func ParseFlags() *Configuration {
 		argKubeConfigFile        = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.")
 		argServiceClusterIPRange = pflag.String("service-cluster-ip-range", "10.96.0.0/12", "The kubernetes service cluster ip range")
 		argNodeSwitch            = pflag.String("node-switch", "join", "The name of node gateway switch which help node to access pod network")
-		argNodeLocalDnsIP        = pflag.String("node-local-dns-ip", "", "If use nodelocaldns the local dns server ip should be set here.")
 		argEncapChecksum         = pflag.Bool("encap-checksum", true, "Enable checksum")
 		argEnablePprof           = pflag.Bool("enable-pprof", false, "Enable pprof")
 		argPprofPort             = pflag.Int("pprof-port", 10665, "The port to get profiling data")
@@ -128,7 +126,6 @@ func ParseFlags() *Configuration {
 		NodeName:                strings.ToLower(*argNodeName),
 		ServiceClusterIPRange:   *argServiceClusterIPRange,
 		NodeSwitch:              *argNodeSwitch,
-		NodeLocalDnsIP:          *argNodeLocalDnsIP,
 		EncapChecksum:           *argEncapChecksum,
 		NetworkType:             *argsNetworkType,
 		CniConfDir:              *argCniConfDir,

pkg/daemon/gateway.go

@@ -2,7 +2,6 @@ package daemon
 
 import (
 	"fmt"
-	"net"
 	"os/exec"
 	"strings"
 
@@ -147,9 +146,7 @@ func (c *Controller) getDefaultVpcSubnetsCIDR(protocol string) ([]string, map[st
 
 	ret := make([]string, 0, len(subnets)+1)
 	subnetMap := make(map[string]string, len(subnets)+1)
-	if c.config.NodeLocalDnsIP != "" && net.ParseIP(c.config.NodeLocalDnsIP) != nil && util.CheckProtocol(c.config.NodeLocalDnsIP) == protocol {
-		ret = append(ret, c.config.NodeLocalDnsIP)
-	}
+
 	for _, subnet := range subnets {
 		if subnet.Spec.Vpc == util.DefaultVpc && (subnet.Spec.Vlan == "" || subnet.Spec.LogicalGateway) && subnet.Spec.CIDRBlock != "" {
 			cidrBlock := getCidrByProtocol(subnet.Spec.CIDRBlock, protocol)

pkg/ovs/ovn-nbctl-legacy.go

@@ -984,10 +984,10 @@ func (c LegacyClient) CreateGatewayACL(ls, pgName, gateway, cidr string) error {
 					egressArgs = append(egressArgs, []string{"--", "--type=port-group", MayExist, "--apply-after-lb", "acl-add", pgName, "from-lport", util.EgressAllowPriority, `nd || nd_ra || nd_rs`, "allow-related"}...)
 				}
 			} else if ls != "" {
-				ingressArgs = []string{"--", MayExist, "acl-add", ls, "to-lport", util.IngressAllowPriority, fmt.Sprintf(`%s.src == %s`, ipSuffix, gw), "allow-related"}
-				egressArgs = []string{"--", MayExist, "--apply-after-lb", "acl-add", ls, "from-lport", util.EgressAllowPriority, fmt.Sprintf(`%s.dst == %s`, ipSuffix, gw), "allow-related"}
+				ingressArgs = []string{"--", MayExist, "acl-add", ls, "to-lport", util.IngressAllowPriority, fmt.Sprintf(`%s.src == %s`, ipSuffix, gw), "allow-stateless"}
+				egressArgs = []string{"--", MayExist, "--apply-after-lb", "acl-add", ls, "from-lport", util.EgressAllowPriority, fmt.Sprintf(`%s.dst == %s`, ipSuffix, gw), "allow-stateless"}
 				if ipSuffix == "ip6" {
-					egressArgs = append(egressArgs, []string{"--", MayExist, "--apply-after-lb", "acl-add", ls, "from-lport", util.EgressAllowPriority, `nd || nd_ra || nd_rs`, "allow-related"}...)
+					egressArgs = append(egressArgs, []string{"--", MayExist, "--apply-after-lb", "acl-add", ls, "from-lport", util.EgressAllowPriority, `nd || nd_ra || nd_rs`, "allow-stateless"}...)
 				}
 			}
 

pkg/util/const.go

@@ -189,9 +189,10 @@ const (
 	IptablesFip = "iptables"
 
 	GatewayRouterPolicyPriority = 29000
+	OvnICPolicyPriority         = 29500
 	NodeRouterPolicyPriority    = 30000
+	NodeLocalDnsPolicyPriority  = 30100
 	SubnetRouterPolicyPriority  = 31000
-	OvnICPolicyPriority         = 29500
 
 	OffloadType  = "offload-port"
 	InternalType = "internal-port"

yamls/kube-ovn-dual-stack.yaml

@@ -90,6 +90,7 @@ spec:
             - --enable-lb-svc=false
             - --keep-vm-ip=true
             - --pod-default-fip-type=
+            - --node-local-dns-ip=
           env:
             - name: ENABLE_SSL
               value: "false"

yamls/kube-ovn-ipv6.yaml

@@ -90,6 +90,7 @@ spec:
             - --enable-lb-svc=false
             - --keep-vm-ip=true
             - --pod-default-fip-type=
+            - --node-local-dns-ip=
           env:
             - name: ENABLE_SSL
               value: "false"

yamls/kube-ovn.yaml

@@ -90,6 +90,7 @@ spec:
             - --enable-lb-svc=false
             - --keep-vm-ip=true
             - --pod-default-fip-type=
+            - --node-local-dns-ip=
           env:
             - name: ENABLE_SSL
               value: "false"