Distributed gateway implement

http://jira.alaudatech.com/browse/OVN-20

Gopkg.toml

@@ -1,3 +1,8 @@
+[prune]
+  non-go = true
+  go-tests = true
+  unused-packages = true
+
 [[constraint]]
   name = "github.com/emicklei/go-restful"
   version = "2.8.0"
@@ -25,3 +30,23 @@
 [[constraint]]
   name = "k8s.io/client-go"
   version = "kubernetes-1.13.2"
+
+[[constraint]]
+  name = "github.com/projectcalico/felix"
+  version = "v3.5.4"
+
+[[override]]
+  name = "github.com/projectcalico/libcalico-go"
+  revision = "3d935c3b8b860224bcedf1aaa02bedaadf0a180c"
+
+[[constraint]]
+  name = "github.com/coreos/go-iptables"
+  version = "v0.4.0"
+
+[[constraint]]
+  name = "github.com/thoas/go-funk"
+  version = "v0.4"
+
+[[constraint]]
+  name = "github.com/juju/errors"
+  branch = "master"

Makefile

@@ -15,7 +15,6 @@ build-go:
 	CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn -ldflags "-w -s" -v ./cmd/cni
 	CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-controller -ldflags "-w -s" -v ./cmd/controller
 	CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-daemon -ldflags "-w -s" -v ./cmd/daemon
-	CGO_ENABLED=0 GOOS=linux go build -o $(PWD)/dist/images/kube-ovn-gateway -ldflags "-w -s" -v ./cmd/gateway
 
 release: build-go
 	docker build -t index.alauda.cn/alaudak8s/kube-ovn-node:`cat VERSION` -f dist/images/Dockerfile.node dist/images/

cmd/controller/controller.go

@@ -11,7 +11,6 @@ import (
 )
 
 func main() {
-	klog.SetOutput(os.Stdout)
 	defer klog.Flush()
 
 	stopCh := signals.SetupSignalHandler()

cmd/daemon/cniserver.go

@@ -1,17 +1,17 @@
 package main
 
 import (
+	"os"
+	"time"
+
 	"bitbucket.org/mathildetech/kube-ovn/pkg/daemon"
 	"bitbucket.org/mathildetech/kube-ovn/pkg/ovs"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog"
 	"k8s.io/sample-controller/pkg/signals"
-	"os"
-	"time"
 )
 
 func main() {
-	klog.SetOutput(os.Stdout)
 	defer klog.Flush()
 	go gc()
 
@@ -23,7 +23,11 @@ func main() {
 
 	stopCh := signals.SetupSignalHandler()
 	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(config.KubeClient, time.Second*30)
-	ctl := daemon.NewController(config, kubeInformerFactory)
+	ctl, err := daemon.NewController(config, kubeInformerFactory)
+	if err != nil {
+		klog.Errorf("create controller failed %v", err)
+		os.Exit(1)
+	}
 	kubeInformerFactory.Start(stopCh)
 	go ctl.Run(stopCh)
 	daemon.RunServer(config)

dist/images/Dockerfile.cni

@@ -1,12 +1,14 @@
 FROM centos:7
 
-RUN yum install -y  \
-        PyYAML bind-utils \
+RUN yum install -y \
+        PyYAML \
+        bind-utils \
         openssl \
         numactl-libs \
         firewalld-filesystem \
         libpcap \
         hostname \
+        ipset \
         iproute strace socat nc \
         unbound unbound-devel && \
         yum clean all
@@ -16,11 +18,11 @@ RUN rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitc
     rpm -i https://github.com/oilbeater/ovs/releases/download/v2.10.1/openvswitch-ovn-host-2.10.1-1.el7.centos.x86_64.rpm
 
 COPY start-cniserver.sh /kube-ovn/start-cniserver.sh
+COPY install-cni.sh /kube-ovn/install-cni.sh
 COPY kube-ovn.conflist /kube-ovn/kube-ovn.conflist
 
 WORKDIR /kube-ovn
 CMD ["sh", "start-cniserver.sh"]
 
 COPY kube-ovn /kube-ovn/kube-ovn
 COPY kube-ovn-daemon /kube-ovn/kube-ovn-daemon
-COPY kube-ovn-gateway /kube-ovn/kube-ovn-gateway

dist/images/install-cni.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+set -u -e
+
+exit_with_error(){
+  echo $1
+  exit 1
+}
+
+CNI_BIN_SRC=/kube-ovn/kube-ovn
+CNI_BIN_DST=/opt/cni/bin/kube-ovn
+CNI_CONF_SRC=/kube-ovn/kube-ovn.conflist
+CNI_CONF_DST=/etc/cni/net.d/kube-ovn.conflist
+
+cp $CNI_BIN_SRC $CNI_BIN_DST || exit_with_error "Failed to copy $CNI_BIN_SRC to $CNI_BIN_DST"
+cp $CNI_CONF_SRC $CNI_CONF_DST || exit_with_error "Failed to copy $CNI_CONF_SRC to $CNI_CONF_DST"

dist/images/start-cniserver.sh

@@ -3,13 +3,10 @@ set -euo pipefail
 
 SOCK=/run/openvswitch/kube-ovn-daemon.sock
 
-cp kube-ovn /opt/cni/bin/kube-ovn
-cp kube-ovn.conflist /etc/cni/net.d/kube-ovn.conflist
-
 if [[ -e "$SOCK" ]]
 then
     echo "previous socket exists, remove and continue"
 	rm ${SOCK}
 fi
 
-./kube-ovn-daemon --ovs-socket=/run/openvswitch/db.sock --bind-socket=${SOCK} $@
+./kube-ovn-daemon --ovs-socket=/run/openvswitch/db.sock --bind-socket=${SOCK} --ovn-nb-host=$OVN_NB_SERVICE_HOST --ovn-sb-host=$OVN_SB_SERVICE_HOST $@

dist/yaml/cm.yaml

@@ -0,0 +1,22 @@
+kind: Configmap
+apiVersion: v1
+metadata:
+  name: ovn-config
+  namespace: kube-ovn
+data:
+  OVN-NB-SOCKET: ""
+  OVN-NB-HOST: "0.0.0.0"
+  OVN-NB-PORT: 6641
+  KUBECONFIG: ""
+  DEFAULT-LS: "ovn-default"
+  DEFAULT-CIDR: "10.16.0.0/16"
+  DEFAULT-GATEWAY: "10.16.0.1"
+  DEFAULT-EXCLUDE-IPS: "10.16.0.0..10.16.0.10"
+  CLUSTER-ROUTER: "ovn-cluster"
+  NODE-SWITCH: "join"
+  NODE-SWITCH-CIDR: "100.64.0.0/16"
+  NODE-SWITCH-GATEWAY: "100.64.0.1"
+  CLUSTER-TCP-LOADBALANCER: "cluster-tcp-loadbalancer"
+  CLUSTER-UDP-LOADBALANCER: "cluster-udp-loadbalancer"
+  BIND-SOCKET: "/var/run/cniserver.sock"
+  OVS-SOCKET: ""

dist/yaml/kube-ovn.yaml

@@ -84,35 +84,41 @@ spec:
       serviceAccountName: ovn
       hostNetwork: true
       hostPID: true
+      initContainers:
+      - name: install-cni
+        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:dev"
+        imagePullPolicy: Always
+        command: ["/kube-ovn/install-cni.sh"]
+        volumeMounts:
+          - mountPath: /etc/cni/net.d
+            name: cni-conf
+          - mountPath: /opt/cni/bin
+            name: cni-bin
       containers:
-        - name: kube-ovn-cni
-          image: "index.alauda.cn/alaudak8s/kube-ovn-cni:dev"
-          imagePullPolicy: Always
-          securityContext:
-            runAsUser: 0
-            privileged: true
-          env:
-            - name: POD_IP
-              valueFrom:
-                fieldRef:
-                  fieldPath: status.podIP
-            - name: KUBE_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: KUBE_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-          volumeMounts:
-            - mountPath: /run/openvswitch
-              name: host-run-ovs
-            - mountPath: /var/run/netns
-              name: host-netns
-            - mountPath:  /etc/cni/net.d
-              name: cni-conf
-            - mountPath: /opt/cni/bin
-              name: cni-bin
+      - name: cni-server
+        image: "index.alauda.cn/alaudak8s/kube-ovn-cni:dev"
+        imagePullPolicy: Always
+        securityContext:
+          runAsUser: 0
+          privileged: true
+        env:
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: KUBE_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: KUBE_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        volumeMounts:
+          - mountPath: /run/openvswitch
+            name: host-run-ovs
+          - mountPath: /var/run/netns
+            name: host-netns
       nodeSelector:
         beta.kubernetes.io/os: "linux"
       volumes:

pkg/controller/config.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"flag"
+
 	"github.com/spf13/pflag"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"

pkg/controller/controller.go

@@ -1,21 +1,22 @@
 package controller
 
 import (
-	"bitbucket.org/mathildetech/kube-ovn/pkg/ovs"
 	"fmt"
+	"time"
+
+	"bitbucket.org/mathildetech/kube-ovn/pkg/ovs"
 	corev1 "k8s.io/api/core/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
-	"k8s.io/client-go/listers/core/v1"
+	v1 "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog"
-	"time"
 )
 
 const controllerAgentName = "ovn-controller"
@@ -36,6 +37,7 @@ type Controller struct {
 	// simultaneously in two different workers.
 	addPodQueue    workqueue.RateLimitingInterface
 	deletePodQueue workqueue.RateLimitingInterface
+	updatePodQueue workqueue.RateLimitingInterface
 
 	namespacesLister     v1.NamespaceLister
 	namespacesSynced     cache.InformerSynced
@@ -91,6 +93,7 @@ func NewController(
 		podsSynced:     podInformer.Informer().HasSynced,
 		addPodQueue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AddPod"),
 		deletePodQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DeletePod"),
+		updatePodQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "UpdatePod"),
 
 		namespacesLister:     namespaceInformer.Lister(),
 		namespacesSynced:     namespaceInformer.Informer().HasSynced,
@@ -118,6 +121,7 @@ func NewController(
 	podInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc:    controller.enqueueAddPod,
 		DeleteFunc: controller.enqueueDeletePod,
+		UpdateFunc: controller.enqueueUpdatePod,
 	})
 
 	namespaceInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
@@ -152,6 +156,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) error {
 	defer utilruntime.HandleCrash()
 	defer c.addPodQueue.ShutDown()
 	defer c.deletePodQueue.ShutDown()
+	defer c.updatePodQueue.ShutDown()
 
 	// Start the informer factories to begin populating the informer caches
 	klog.Info("Starting OVN controller")
@@ -167,6 +172,7 @@ func (c *Controller) Run(stopCh <-chan struct{}) error {
 	// Launch workers to process resources
 	go wait.Until(c.runAddPodWorker, time.Second, stopCh)
 	go wait.Until(c.runDeletePodWorker, time.Second, stopCh)
+	go wait.Until(c.runUpdatePodWorker, time.Second, stopCh)
 
 	go wait.Until(c.runAddNamespaceWorker, time.Second, stopCh)
 	go wait.Until(c.runDeleteNamespaceWorker, time.Second, stopCh)

pkg/controller/node.go

@@ -121,7 +121,7 @@ func (c *Controller) handleAddNode(key string) error {
 	}
 
 	nodeAddr := getNodeInternalIP(node)
-	err = c.ovnClient.AddStaticRouter(nodeAddr, strings.Split(nic.IpAddress, "/")[0], c.config.ClusterRouter)
+	err = c.ovnClient.AddStaticRouter("", nodeAddr, strings.Split(nic.IpAddress, "/")[0], c.config.ClusterRouter)
 	if err != nil {
 		return err
 	}