feat: use daemon ovn-nbctl to improve performance and cleanup unused …

…dns code

cmd/controller/controller.go

@@ -1,7 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/alauda/kube-ovn/pkg/controller"
@@ -21,6 +24,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	go loopOvnNbctlDaemon(config)
+
 	err = controller.InitClusterRouter(config)
 	if err != nil {
 		klog.Errorf("init cluster router failed %v", err)
@@ -33,12 +38,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	err = controller.InitDnsTable(config)
-	if err != nil {
-		klog.Errorf("init dns table failed %v", err)
-		os.Exit(1)
-	}
-
 	err = controller.InitNodeSwitch(config)
 	if err != nil {
 		klog.Errorf("init node switch failed %v", err)
@@ -56,3 +55,32 @@ func main() {
 	kubeInformerFactory.Start(stopCh)
 	ctl.Run(stopCh)
 }
+
+func loopOvnNbctlDaemon(config *controller.Configuration) {
+	for {
+		daemonSocket := os.Getenv("OVN_NB_DAEMON")
+		time.Sleep(5 * time.Second)
+
+		if _, err := os.Stat(daemonSocket); os.IsNotExist(err) || daemonSocket == "" {
+			startOvnNbctlDaemon(config.OvnNbHost, config.OvnNbPort)
+		}
+	}
+}
+
+func startOvnNbctlDaemon(nbHost string, nbPort int) (string, error) {
+	klog.Infof("start ovn-nbctl daemon")
+	output, err := exec.Command(
+		"ovn-nbctl",
+		fmt.Sprintf("--db=tcp:%s:%d", nbHost, nbPort),
+		"--pidfile",
+		"--detach",
+	).CombinedOutput()
+	if err != nil {
+		klog.Errorf("start ovn-nbctl daemon failed, %s", string(output))
+		return "", err
+	} else {
+		daemonSocket := strings.TrimSpace(string(output))
+		os.Setenv("OVN_NB_DAEMON", daemonSocket)
+		return daemonSocket, nil
+	}
+}

dist/images/Dockerfile.controller

@@ -20,7 +20,9 @@ RUN rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitc
     rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-central-2.10.1-2.el7.centos.x86_64.rpm && \
     rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-host-2.10.1-1.el7.centos.x86_64.rpm
 
+RUN mkdir -p /var/run/openvswitch
 WORKDIR /kube-ovn
+
 CMD ["sh", "start-controller.sh"]
 
 COPY start-controller.sh /kube-ovn/start-controller.sh

dist/images/start-controller.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 set -euo pipefail
-
+export OVN_NB_DAEMON=$(ovn-nbctl --db=tcp:${OVN_NB_SERVICE_HOST}:${OVN_NB_SERVICE_PORT} --pidfile --detach)
 ./kube-ovn-controller --ovn-nb-host=${OVN_NB_SERVICE_HOST} --ovn-nb-port=${OVN_NB_SERVICE_PORT} $@

pkg/controller/init.go

@@ -117,14 +117,3 @@ func InitLoadBalancer(config *Configuration) error {
 	}
 	return nil
 }
-
-func InitDnsTable(config *Configuration) error {
-	client := ovs.NewClient(config.OvnNbHost, config.OvnNbPort, "", 0, config.ClusterRouter, config.ClusterTcpLoadBalancer, config.ClusterUdpLoadBalancer, config.NodeSwitch, config.NodeSwitchCIDR)
-	uuid, err := client.CreateDnsTable()
-	if err != nil {
-		return err
-	}
-	ovs.GlobalDnsTable = uuid
-	klog.Infof("dns table is %s", uuid)
-	return nil
-}

pkg/controller/namespace.go

@@ -174,9 +174,6 @@ func (c *Controller) handleAddNamespace(key string) error {
 		klog.Infof("namespace %s use default logical switch %s", key, c.config.DefaultLogicalSwitch)
 		return nil
 	}
-	if err != nil {
-		return err
-	}
 
 	// skip creation if switch already exists
 	exist := false

pkg/controller/service.go

@@ -141,8 +141,6 @@ func (c *Controller) handleAddService(key string) error {
 			svc.SetFinalizers(removeString(svc.Finalizers, util.ServiceAnnotation))
 			_, err = c.config.KubeClient.CoreV1().Services(namespace).Update(svc)
 		}
-	} else {
-		err = c.ovnClient.AddDnsRecord(svcDomain(name, namespace), []string{ip})
 	}
 
 	return err
@@ -207,12 +205,6 @@ func (c *Controller) handleUpdateService(key string) error {
 				return err
 			}
 		}
-
-		err = c.ovnClient.DeleteDnsRecord(svcDomain(name, namespace))
-		if err != nil {
-			klog.Errorf("failed to delete dns %s , %v", svcDomain(name, namespace), err)
-			return err
-		}
 	} else {
 		// for service update
 		klog.Infof("update service %s/%s", namespace, name)
@@ -299,7 +291,3 @@ func removeString(slice []string, s string) (result []string) {
 	}
 	return
 }
-
-func svcDomain(svc, namespace string) string {
-	return fmt.Sprintf("%s.%s.svc.cluster.local", svc, namespace)
-}

pkg/ovs/ovn-nbctl.go

@@ -11,11 +11,7 @@ import (
 	"k8s.io/klog"
 )
 
-func (c Client) ovnNbCommand(arg ...string) (string, error) {
-	cmdArgs := []string{fmt.Sprintf("--db=%s", c.OvnNbAddress)}
-	cmdArgs = append(cmdArgs, arg...)
-	klog.V(5).Infof("execute %s command %s", OvnNbCtl, strings.Join(cmdArgs, " "))
-
+func (c Client) ovnNbCommand(cmdArgs ...string) (string, error) {
 	start := time.Now()
 	raw, err := exec.Command(OvnNbCtl, cmdArgs...).CombinedOutput()
 	elapsed := float64((time.Since(start)) / time.Millisecond)
@@ -189,13 +185,6 @@ func (c Client) CreateLogicalSwitch(ls, subnet, gateway, excludeIps string) erro
 			klog.Errorf("failed to add cluster udp lb to %s, %v", ls, err)
 			return err
 		}
-
-		// DO NOT add dns table until we find the reason why we can not resolve external domains
-		//err = c.AddDnsTableToLogicalSwitch(ls)
-		//if err != nil {
-		//	klog.Errorf("failed to add cluster dns to %s, %v", ls, err)
-		//	return err
-		//}
 	}
 
 	return nil
@@ -338,54 +327,6 @@ func (c Client) GetLoadBalancerVips(lb string) (map[string]string, error) {
 	return result, err
 }
 
-func (c Client) CreateDnsTable() (string, error) {
-	output, err := c.ovnNbCommand("--data=bare", "--no-heading", "--columns=_uuid", fmt.Sprintf("--db=%s", c.OvnNbAddress),
-		"find", "DNS", "external_ids:name=ovn")
-	if err != nil {
-		return "", nil
-	}
-	if output != "" {
-		return output, nil
-	} else {
-		_, err := c.ovnNbCommand("create", "DNS", "external_ids:name=ovn")
-		if err != nil {
-			return "", nil
-		}
-		output, err := c.ovnNbCommand("--data=bare", "--no-heading", "--columns=_uuid", fmt.Sprintf("--db=%s", c.OvnNbAddress),
-			"find", "DNS", "external_ids:name=ovn")
-		if err != nil {
-			return "", nil
-		}
-		return output, nil
-	}
-}
-
-func (c Client) AddDnsRecord(domain string, addresses []string) error {
-	_, err := c.ovnNbCommand("set", "DNS", GlobalDnsTable, fmt.Sprintf("records:%s=%s", domain, strings.Join(addresses, ",")))
-	return err
-}
-
-func (c Client) DeleteDnsRecord(domain string) error {
-	_, err := c.ovnNbCommand(IfExists, "remove", "DNS", GlobalDnsTable, "records", domain)
-	return err
-}
-
-func (c Client) GetDnsRecords() (map[string]string, error) {
-	output, err := c.ovnNbCommand("--data=bare", "--no-heading",
-		"get", "dns", GlobalDnsTable, "records")
-	if err != nil {
-		return nil, err
-	}
-	result := map[string]string{}
-	err = json.Unmarshal([]byte(strings.Replace(output, "=", ":", -1)), &result)
-	return result, err
-}
-
-func (c Client) AddDnsTableToLogicalSwitch(ls string) error {
-	_, err := c.ovnNbCommand("add", "logical_switch", ls, "dns_records", GlobalDnsTable)
-	return err
-}
-
 func (c Client) CleanLogicalSwitchAcl(ls string) error {
 	_, err := c.ovnNbCommand("acl-del", ls)
 	return err

pkg/ovs/ovn.go

@@ -31,8 +31,6 @@ const (
 	PolicySrcIP = "src-ip"
 )
 
-var GlobalDnsTable string
-
 func NewClient(ovnNbHost string, ovnNbPort int, ovnSbHost string, ovnSbPort int, clusterRouter, clusterTcpLoadBalancer, clusterUdpLoadBalancer, nodeSwitch, nodeSwitchCIDR string) *Client {
 	return &Client{
 		OvnNbAddress:           fmt.Sprintf("tcp:%s:%d", ovnNbHost, ovnNbPort),

pkg/util/net.go

@@ -2,9 +2,7 @@ package util
 
 import (
 	"fmt"
-	"math/big"
 	"math/rand"
-	"net"
 	"time"
 )
 
@@ -15,20 +13,3 @@ func GenerateMac() string {
 	mac := fmt.Sprintf("%s:%02X:%02X:%02X", prefix, newRand.Intn(255), newRand.Intn(255), newRand.Intn(255))
 	return mac
 }
-
-// NextIP returns IP incremented by 1
-func NextIP(ip net.IP) net.IP {
-	i := ipToInt(ip)
-	return intToIP(i.Add(i, big.NewInt(1)))
-}
-
-func ipToInt(ip net.IP) *big.Int {
-	if v := ip.To4(); v != nil {
-		return big.NewInt(0).SetBytes(v)
-	}
-	return big.NewInt(0).SetBytes(ip.To16())
-}
-
-func intToIP(i *big.Int) net.IP {
-	return net.IP(i.Bytes())
-}