feat: support traffic mirror

cmd/controller/controller.go

@@ -21,37 +21,28 @@ func main() {
 
 	config, err := controller.ParseFlags()
 	if err != nil {
-		klog.Errorf("parse config failed %v", err)
-		os.Exit(1)
+		klog.Fatalf("parse config failed %v", err)
 	}
 
 	go loopOvnNbctlDaemon(config)
 	go func() {
 		klog.Fatal(http.ListenAndServe(fmt.Sprintf("localhost:%d", config.PprofPort), nil))
 	}()
 
-	err = controller.InitClusterRouter(config)
-	if err != nil {
-		klog.Errorf("init cluster router failed %v", err)
-		os.Exit(1)
+	if err = controller.InitClusterRouter(config); err != nil {
+		klog.Fatalf("init cluster router failed %v", err)
 	}
 
-	err = controller.InitLoadBalancer(config)
-	if err != nil {
-		klog.Errorf("init load balancer failed %v", err)
-		os.Exit(1)
+	if err = controller.InitLoadBalancer(config); err != nil {
+		klog.Fatalf("init load balancer failed %v", err)
 	}
 
-	err = controller.InitNodeSwitch(config)
-	if err != nil {
-		klog.Errorf("init node switch failed %v", err)
-		os.Exit(1)
+	if err = controller.InitNodeSwitch(config); err != nil {
+		klog.Fatalf("init node switch failed %v", err)
 	}
 
-	err = controller.InitDefaultLogicalSwitch(config)
-	if err != nil {
-		klog.Errorf("init default switch failed %v", err)
-		os.Exit(1)
+	if err = controller.InitDefaultLogicalSwitch(config); err != nil {
+		klog.Fatalf("init default switch failed %v", err)
 	}
 
 	ctl := controller.NewController(config)

cmd/daemon/cniserver.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net/http"
 	_ "net/http/pprof"
-	"os"
 	"time"
 
 	"github.com/alauda/kube-ovn/pkg/daemon"
@@ -20,22 +19,24 @@ func main() {
 
 	config, err := daemon.ParseFlags()
 	if err != nil {
-		klog.Errorf("parse config failed %v", err)
-		os.Exit(1)
+		klog.Fatalf("parse config failed %v", err)
 	}
 
-	err = daemon.InitNodeGateway(config)
-	if err != nil {
-		klog.Errorf("init node gateway failed %v", err)
-		os.Exit(1)
+	if config.EnableMirror {
+		if err = daemon.InitMirror(config); err != nil {
+			klog.Fatalf("failed to init mirror nic, %v", err)
+		}
+	}
+
+	if err = daemon.InitNodeGateway(config); err != nil {
+		klog.Fatalf("init node gateway failed %v", err)
 	}
 
 	stopCh := signals.SetupSignalHandler()
 	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(config.KubeClient, time.Second*30)
 	ctl, err := daemon.NewController(config, kubeInformerFactory)
 	if err != nil {
-		klog.Errorf("create controller failed %v", err)
-		os.Exit(1)
+		klog.Fatalf("create controller failed %v", err)
 	}
 	kubeInformerFactory.Start(stopCh)
 	go ctl.Run(stopCh)

docs/install.md

@@ -54,7 +54,9 @@ For high-available ovn db, see [high available](high-available.md)
 
 ```bash
     --iface: The iface used to inter-host pod communication, default: the default route iface
-    --mtu: The MTU used by pod iface, default: iface MTU - 55
+    --mtu: The MTU used by pod iface, default: iface MTU - 58
+    --enable-mirror: Enable traffic mirror, default: false
+    --mirror-iface: The mirror nic name that will be created by kube-ovn, default: mirror0
     --service-cluster-ip-range: The kubernetes service cluster ip range, default: 10.96.0.0/12
     --kubeconfig: Path to kubeconfig file with authorization and master location information. If not set use the inCluster token
 ```

docs/mirror.md

@@ -0,0 +1,17 @@
+# Traffic Mirror
+
+Kube-OVN support traffic mirroring that duplicates pod nic send/receive network packets to a dedicated host nic. Administrator or developer can then use it to monitor network traffic or diagnose network problems.
+
+![alt text](mirror.png "kube-ovn mirror")
+
+## Enable Traffic Mirror
+
+Traffic mirror is disabled by default, you should add cmd args in cni-server when installing kube-ovn to enabled it.
+- `--enable-mirror=true`: enable traffic mirror
+- `--mirror-iface=mirror0`: kube-ovn will create an ovs internal port on every node to mirror pod traffic on that node. If not set will use mirror0 as the default port name
+
+Then you can use tcpdump or other tools to diagnose traffic from interface mirror0:
+
+```bash
+tcpdump -ni mirror0
+```

pkg/daemon/config.go

@@ -26,6 +26,8 @@ import (
 type Configuration struct {
 	Iface                 string
 	MTU                   int
+	EnableMirror          bool
+	MirrorNic             string
 	BindSocket            string
 	OvsSocket             string
 	KubeConfigFile        string
@@ -41,6 +43,8 @@ func ParseFlags() (*Configuration, error) {
 	var (
 		argIface                 = pflag.String("iface", "", "The iface used to inter-host pod communication, default: the default route iface")
 		argMTU                   = pflag.Int("mtu", 0, "The MTU used by pod iface, default: iface MTU - 55")
+		argEnableMirror          = pflag.Bool("enable-mirror", false, "Enable traffic mirror, default: false")
+		argMirrorNic             = pflag.String("mirror-iface", "mirror0", "The mirror nic name that will be created by kube-ovn, default: mirror0")
 		argBindSocket            = pflag.String("bind-socket", "/var/run/cniserver.sock", "The socket daemon bind to.")
 		argOvsSocket             = pflag.String("ovs-socket", "", "The socket to local ovs-server")
 		argKubeConfigFile        = pflag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information. If not set use the inCluster token.")
@@ -78,6 +82,8 @@ func ParseFlags() (*Configuration, error) {
 	config := &Configuration{
 		Iface:                 *argIface,
 		MTU:                   *argMTU,
+		EnableMirror:          *argEnableMirror,
+		MirrorNic:             *argMirrorNic,
 		BindSocket:            *argBindSocket,
 		OvsSocket:             *argOvsSocket,
 		KubeConfigFile:        *argKubeConfigFile,

pkg/daemon/init.go

@@ -35,3 +35,7 @@ func InitNodeGateway(config *Configuration) error {
 	}
 	return configureNodeNic(portName, ipAddr, macAddr, gw, config.MTU)
 }
+
+func InitMirror(config *Configuration) error {
+	return configureMirror(config.MirrorNic, config.MTU)
+}

pkg/daemon/ovs.go

@@ -201,7 +201,7 @@ func configureNodeNic(portName, ip, mac, gw string, mtu int) error {
 
 	err = netlink.LinkSetMTU(nodeLink, mtu)
 	if err != nil {
-		return fmt.Errorf("can not set mtu %v", err)
+		return fmt.Errorf("can not set node nic mtu %v", err)
 	}
 
 	if nodeLink.Attrs().OperState != netlink.OperUp {
@@ -216,3 +216,36 @@ func configureNodeNic(portName, ip, mac, gw string, mtu int) error {
 	klog.Infof("ping gw result is: \n %s", string(output))
 	return nil
 }
+
+func configureMirror(portName string, mtu int) error {
+	raw, err := exec.Command(
+		"ovs-vsctl", "--may-exist", "add-port", "br-int", portName, "--",
+		"set", "interface", portName, "type=internal", "--",
+		"clear", "bridge", "br-int", "mirrors", "--",
+		"--id=@mirror0", "get", "port", portName, "--",
+		"--id=@m", "create", "mirror", "name=m0", "select_all=true", "output_port=@mirror0", "--",
+		"add", "bridge", "br-int", "mirrors", "@m").CombinedOutput()
+	if err != nil {
+		klog.Errorf("failed to configure mirror nic %s %s", portName, string(raw))
+		return fmt.Errorf(string(raw))
+	}
+
+	mirrorLink, err := netlink.LinkByName(portName)
+	if err != nil {
+		return fmt.Errorf("can not find mirror nic %s %v", portName, err)
+	}
+
+	err = netlink.LinkSetMTU(mirrorLink, mtu)
+	if err != nil {
+		return fmt.Errorf("can not set mirror nic mtu %v", err)
+	}
+
+	if mirrorLink.Attrs().OperState != netlink.OperUp {
+		err = netlink.LinkSetUp(mirrorLink)
+		if err != nil {
+			return fmt.Errorf("can not set mirror nic %s up %v", portName, err)
+		}
+	}
+
+	return nil
+}

yamls/kube-ovn.yaml

@@ -38,7 +38,7 @@ spec:
       hostNetwork: true
       containers:
         - name: kube-ovn-controller
-          image: "index.alauda.cn/alaudak8s/kube-ovn-controller:v0.5.0"
+          image: "index.alauda.cn/alaudak8s/kube-ovn-controller:v0.6.0-pre"
           imagePullPolicy: Always
           command:
           - /kube-ovn/start-controller.sh
@@ -108,7 +108,7 @@ spec:
       hostPID: true
       initContainers:
       - name: install-cni
-        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.5.0"
+        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.6.0-pre"
         imagePullPolicy: Always
         command: ["/kube-ovn/install-cni.sh"]
         volumeMounts:
@@ -118,8 +118,13 @@ spec:
             name: cni-bin
       containers:
       - name: cni-server
-        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.5.0"
+        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:v0.6.0-pre"
         imagePullPolicy: Always
+        command:
+          - sh
+          - /kube-ovn/start-cniserver.sh
+        args:
+          - --enable-mirror=false
         securityContext:
           runAsUser: 0
           privileged: true

yamls/ovn.yaml

@@ -170,7 +170,7 @@ spec:
       hostNetwork: true
       containers:
         - name: ovn-central
-          image: "index.alauda.cn/alaudak8s/kube-ovn-db:v0.5.0"
+          image: "index.alauda.cn/alaudak8s/kube-ovn-db:v0.6.0-pre"
           imagePullPolicy: Always
           env:
             - name: POD_IP
@@ -257,7 +257,7 @@ spec:
       hostPID: true
       containers:
         - name: openvswitch
-          image: "index.alauda.cn/alaudak8s/kube-ovn-node:v0.5.0"
+          image: "index.alauda.cn/alaudak8s/kube-ovn-node:v0.6.0-pre"
           imagePullPolicy: Always
           securityContext:
             runAsUser: 0