add container compile and insmod

kubeovn · Feb 14, 2022 · e9c9b1c · e9c9b1c
1 parent 0fd564e
commit e9c9b1c
Show file tree

Hide file tree

Showing 13 changed files with 819 additions and 2 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -75,6 +75,18 @@ jobs:
           name: vpc-nat-gateway
           path: vpc-nat-gateway.tar
 
+      - name: Upload centos7-compile image to artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos7-compile
+          path: centos7-compile.tar
+
+      - name: Upload centos8-compile image to artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: centos8-compile
+          path: centos8-compile.tar
+
   single-e2e:
     needs: build
     name: 1-master-e2e
@@ -1122,10 +1134,22 @@ jobs:
         with:
           name: vpc-nat-gateway
 
+      - name: Download centos7-compile image
+        uses: actions/Download-artifact@v2
+        with:
+          name: centos7-compile
+
+      - name: Download centos8-compile image
+        uses: actions/Download-artifact@v2
+        with:
+          name: centos8-compile
+
       - name: Load Image
         run: |
           docker load --input kube-ovn.tar
           docker load --input vpc-nat-gateway.tar
+          docker load --input centos7-compile.tar
+          docker load --input centos8-compile.tar
 
       - name: Security Scan
         run: |
@@ -1151,8 +1175,16 @@ jobs:
           docker tag kubeovn/kube-ovn:$TAG kubeovn/kube-ovn:$TAG-x86
           docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
           docker tag kubeovn/vpc-nat-gateway:$TAG kubeovn/vpc-nat-gateway:$TAG-x86
+          docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile-dev:$TAG-x86
+          docker tag kubeovn/centos7-compile:$TAG kubeovn/centos7-compile:$TAG-x86
+          docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile-dev:$TAG-x86
+          docker tag kubeovn/centos8-compile:$TAG kubeovn/centos8-compile:$TAG-x86
           docker images
           docker push kubeovn/kube-ovn:$TAG-x86
           docker push kubeovn/kube-ovn-dev:$COMMIT-x86
           docker push kubeovn/vpc-nat-gateway:$TAG-x86
           docker push kubeovn/vpc-nat-gateway-dev:$COMMIT-x86
+          docker push kubeovn/centos7-compile:$TAG-x86
+          docker push kubeovn/centos7-compile-dev:$TAG-x86
+          docker push kubeovn/centos8-compile:$TAG-x86
+          docker push kubeovn/centos8-compile-dev:$TAG-x86
diff --git a/Makefile b/Makefile
@@ -51,6 +51,8 @@ base-arm64:
 release: lint build-go
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
 
 .PHONY: release-arm
 release-arm: build-go-arm
@@ -69,6 +71,8 @@ push-release: release
 tar:
 	docker save $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o kube-ovn.tar
 	docker save $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o vpc-nat-gateway.tar
+	docker save $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o centos7-compile.tar
+	docker save $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o centos8-compile.tar
 
 .PHONY: base-tar-amd64
 base-tar-amd64:

diff --git a/dist/images/compile/centos7/Dockerfile b/dist/images/compile/centos7/Dockerfile
@@ -0,0 +1,7 @@
+FROM centos:7
+
+RUN yum install -y gcc elfutils-libelf-devel make perl python3 autoconf automake libtool rpm-build openssl-devel git \
+    && git clone -b branch-2.16 --depth=1 https://github.com/openvswitch/ovs.git /ovs/ \
+    && yum erase -y git && yum clean all
+COPY /*  /fastpath/
+WORKDIR /fastpath
diff --git a/dist/images/compile/centos8/Dockerfile b/dist/images/compile/centos8/Dockerfile
@@ -0,0 +1,12 @@
+FROM centos:8
+
+COPY /  /fastpath/
+RUN find /etc/yum.repos.d/ -type f -exec sed -i 's/mirrorlist=/#mirrorlist=/g' {} + \
+    && find /etc/yum.repos.d/ -type f -exec sed -i 's/#baseurl=/baseurl=/g' {} + \
+    && find /etc/yum.repos.d/ -type f -exec sed -i 's/mirror.centos.org/vault.centos.org/g' {} + \
+    && yum install -y gcc elfutils-libelf-devel make perl python3 autoconf automake libtool rpm-build openssl-devel git \
+    && git clone -b branch-2.16 --depth=1 https://github.com/openvswitch/ovs.git /ovs/ \
+    && yum erase -y git && yum clean all
+COPY /  /fastpath/
+RUN rm -f /fastpath/kube_ovn_fastpath.c && mv /fastpath/4.18/kube_ovn_fastpath.c /fastpath/kube_ovn_fastpath.c
+WORKDIR /fastpath
diff --git a/dist/images/install.sh b/dist/images/install.sh
@@ -74,6 +74,10 @@ DPDK_VERSION=""
 DPDK_CPU="1000m"                        # Default CPU configuration for if --dpdk-cpu flag is not included
 DPDK_MEMORY="2Gi"                       # Default Memory configuration for it --dpdk-memory flag is not included
 
+# performance
+MODULES="kube_ovn_fastpath.ko"
+RPMS="openvswitch-kmod"
+
 display_help() {
     echo "Usage: $0 [option...]"
     echo
@@ -1993,6 +1997,10 @@ spec:
             valueFrom:
               fieldRef:
                 fieldPath: spec.nodeName
+          - name: MODULES
+            value: $MODULES
+          - name: RPMS
+            value: $RPMS
         volumeMounts:
           - mountPath: /etc/openvswitch
             name: systemid
@@ -2009,6 +2017,8 @@ spec:
             name: kube-ovn-log
           - mountPath: /etc/localtime
             name: localtime
+          - mountPath: /tmp
+            name: tmp
         livenessProbe:
           failureThreshold: 3
           initialDelaySeconds: 30
@@ -2059,6 +2069,9 @@ spec:
         - name: localtime
           hostPath:
             path: /etc/localtime
+        - name: tmp
+          hostPath:
+            path: /tmp
 
 ---
 kind: DaemonSet
@@ -2411,6 +2424,8 @@ set -euo pipefail
 KUBE_OVN_NS=kube-system
 OVN_NB_POD=
 OVN_SB_POD=
+KUBE_OVN_VERSION=
+REGISTRY="kubeovn"
 
 showHelp(){
   echo "kubectl ko {subcommand} [option...]"
@@ -2425,6 +2440,7 @@ showHelp(){
   echo "  tcpdump {namespace/podname} [tcpdump options ...]     capture pod traffic"
   echo "  trace {namespace/podname} {target ip address} {icmp|tcp|udp} [target tcp or udp port]    trace ovn microflow of specific packet"
   echo "  diagnose {all|node} [nodename]    diagnose connectivity of all nodes or a specific node"
+  echo "  tuning {install-fastpath|local-install-fastpath|remove-fastpath|install-stt|local-install-stt|remove-stt} {centos7|centos8}} [kernel-devel-version]  deploy  kernel optimisation components to the system"
   echo "  reload restart all kube-ovn components"
 }
 
@@ -2771,6 +2787,12 @@ getOvnCentralPod(){
       exit 1
     fi
     OVN_SB_POD=$SB_POD
+    VERSION=$(kubectl  -n kube-system get pods -l ovn-sb-leader=true -o yaml | grep  "image: $REGISTRY/kube-ovn:" | head -n 1 | awk -F ':' '{print $3}')
+    if [ -z "$VERSION" ]; then
+          echo "kubeovn version not exists"
+          exit 1
+        fi
+    KUBE_OVN_VERSION=$VERSION
 }
 
 checkDaemonSet(){
@@ -2873,6 +2895,141 @@ dbtool(){
   esac
 }
 
+tuning(){
+  action="$1"; shift
+  sys="$1"; shift
+  case $action in
+    install-fastpath)
+      case $sys in
+        centos7)
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh  centos install"
+          while [ ! -f /tmp/kube_ovn_fastpath.ko ];
+          do
+            sleep 1
+          done
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/
+          done
+          ;;
+        centos8)
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp/:/tmp/ $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh  centos install"
+          while [ ! -f /tmp/kube_ovn_fastpath.ko ];
+          do
+            sleep 1
+          done
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/
+          done
+          ;;
+        *)
+          echo "unknown system $sys"
+      esac
+      ;;
+    local-install-fastpath)
+      case $sys in
+        centos7)
+          # shellcheck disable=SC2145
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@"
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/
+          done
+          ;;
+        centos8)
+          # shellcheck disable=SC2145
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh centos local-install $@"
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            kubectl cp /tmp/kube_ovn_fastpath.ko kube-system/"$i":/tmp/
+          done
+          ;;
+        *)
+          echo "unknown system $sys"
+      esac
+      ;;
+    remove-fastpath)
+      case $sys in
+        centos)
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            kubectl -n kube-system exec "$i" -- rm -f /tmp/kube_ovn_fastpath.ko
+          done
+          ;;
+        *)
+          echo "unknown system $sys"
+      esac
+      ;;
+    install-stt)
+      case $sys in
+        centos7)
+          # shellcheck disable=SC2145
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install"
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            for k in /tmp/*.rpm; do
+              kubectl cp "$k" kube-system/"$i":/tmp/
+            done
+          done
+          ;;
+        centos8)
+          # shellcheck disable=SC2145
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt install"
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            for k in /tmp/*.rpm; do
+              kubectl cp "$k" kube-system/"$i":/tmp/
+            done
+          done
+          ;;
+        *)
+          echo "unknown system $sys"
+      esac
+      ;;
+    local-install-stt)
+      case $sys in
+        centos7)
+          # shellcheck disable=SC2145
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos7-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@"
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            for k in /tmp/*.rpm; do
+              kubectl cp "$k" kube-system/"$i":/tmp/
+            done
+          done
+          ;;
+        centos8)
+          # shellcheck disable=SC2145
+          docker run -it --privileged -v /lib/modules:/lib/modules -v /usr/src:/usr/src -v /tmp:/tmp $REGISTRY/centos8-compile:"$KUBE_OVN_VERSION" bash -c "./module.sh stt local-install $@"
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            for k in /tmp/*.rpm; do
+              kubectl cp "$k" kube-system/"$i":/tmp/
+            done
+          done
+          ;;
+        *)
+          echo "unknown system $sys"
+      esac
+      ;;
+    remove-stt)
+      case $sys in
+        centos)
+          for i in $(kubectl -n kube-system get pods | grep ovn-cni | awk '{print $1}');
+          do
+            kubectl -n kube-system exec "$i" -- rm -f /tmp/openvswitch-kmod*.rpm
+          done
+          ;;
+        *)
+          echo "unknown system $sys"
+      esac
+      ;;
+    *)
+      echo "unknown action $action"
+  esac
+}
+
 reload(){
   kubectl delete pod -n kube-system -l app=ovn-central
   kubectl rollout status deployment/ovn-central -n kube-system
@@ -2921,8 +3078,11 @@ case $subcommand in
   reload)
     reload
     ;;
+  tuning)
+    tuning "$@"
+    ;;
   *)
-    showHelp
+  showHelp
     ;;
 esac