support dpdk (#1317)

* support dpdk hybrid node

Co-authored-by: luoyunhe <luoyunhe@chinatelecom.cn>

.github/workflows/build-kube-ovn-base-dpdk.yaml

@@ -0,0 +1,56 @@
+name: Build Base DPDK
+on: workflow_dispatch
+
+jobs:
+  build-amd64:
+    name: Build AMD64
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Build
+        run: |
+          make base-amd64-dpdk
+          make base-tar-amd64-dpdk
+
+      - name: Upload image to artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: image-amd64-dpdk
+          path: image-amd64-dpdk.tar
+
+
+  push:
+    needs:
+      - build-amd64
+    name: push
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+
+      - name: Download image
+        uses: actions/download-artifact@v2
+        with:
+          name: image-amd64-dpdk
+
+      - name: Load Image
+        run: |
+          docker load --input image-amd64-dpdk.tar
+
+      - name: Push
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          COMMIT: ${{ github.sha }}
+        run: |
+          cat VERSION
+          TAG=$(cat VERSION)
+          echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+          docker images
+          docker tag kubeovn/kube-ovn-base:$TAG-amd64-dpdk kubeovn/kube-ovn-base:$TAG-dpdk
+          docker push kubeovn/kube-ovn-base:$TAG-dpdk

Makefile

@@ -44,13 +44,18 @@ build-dpdk:
 base-amd64:
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-amd64 -o type=docker -f dist/images/Dockerfile.base dist/images/
 
+.PHONY: base-amd64-dpdk
+base-amd64-dpdk:
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-amd64-dpdk -o type=docker -f dist/images/Dockerfile.base-dpdk dist/images/
+
 .PHONY: base-arm64
 base-arm64:
 	docker buildx build --platform linux/arm64 --build-arg ARCH=arm64 -t $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-arm64 -o type=docker -f dist/images/Dockerfile.base dist/images/
 
 .PHONY: release
 release: lint build-go
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
+	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG)-dpdk -o type=docker -f dist/images/Dockerfile.dpdk dist/images/
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
 	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos7-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos7/Dockerfile fastpath/
 #	docker buildx build --platform linux/amd64 --build-arg ARCH=amd64 -t $(REGISTRY)/centos8-compile:$(RELEASE_TAG) -o type=docker -f dist/images/compile/centos8/Dockerfile fastpath/
@@ -79,6 +84,10 @@ tar:
 base-tar-amd64:
 	docker save $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-amd64 -o image-amd64.tar
 
+.PHONY: base-tar-amd64-dpdk
+base-tar-amd64-dpdk:
+	docker save $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-amd64-dpdk -o image-amd64-dpdk.tar
+
 .PHONY: base-tar-arm64
 base-tar-arm64:
 	docker save $(REGISTRY)/kube-ovn-base:$(RELEASE_TAG)-arm64 -o image-arm64.tar

cmd/cni/cni.go

@@ -48,16 +48,18 @@ func cmdAdd(args *skel.CmdArgs) error {
 
 	client := request.NewCniServerClient(netConf.ServerSocket)
 	response, err := client.Add(request.CniRequest{
-		CniType:      netConf.Type,
-		PodName:      podName,
-		PodNamespace: podNamespace,
-		ContainerID:  args.ContainerID,
-		NetNs:        args.Netns,
-		IfName:       args.IfName,
-		Provider:     netConf.Provider,
-		Routes:       netConf.Routes,
-		DeviceID:     netConf.DeviceID,
-		VfDriver:     netConf.VfDriver,
+		CniType:                   netConf.Type,
+		PodName:                   podName,
+		PodNamespace:              podNamespace,
+		ContainerID:               args.ContainerID,
+		NetNs:                     args.Netns,
+		IfName:                    args.IfName,
+		Provider:                  netConf.Provider,
+		Routes:                    netConf.Routes,
+		DeviceID:                  netConf.DeviceID,
+		VfDriver:                  netConf.VfDriver,
+		VhostUserSocketVolumeName: netConf.VhostUserSocketVolumeName,
+		VhostUserSocketName:       netConf.VhostUserSocketName,
 	})
 	if err != nil {
 		return err
@@ -131,14 +133,15 @@ func cmdDel(args *skel.CmdArgs) error {
 	}
 
 	return client.Del(request.CniRequest{
-		CniType:      netConf.Type,
-		PodName:      podName,
-		PodNamespace: podNamespace,
-		ContainerID:  args.ContainerID,
-		NetNs:        args.Netns,
-		IfName:       args.IfName,
-		Provider:     netConf.Provider,
-		DeviceID:     netConf.DeviceID,
+		CniType:                   netConf.Type,
+		PodName:                   podName,
+		PodNamespace:              podNamespace,
+		ContainerID:               args.ContainerID,
+		NetNs:                     args.Netns,
+		IfName:                    args.IfName,
+		Provider:                  netConf.Provider,
+		DeviceID:                  netConf.DeviceID,
+		VhostUserSocketVolumeName: netConf.VhostUserSocketVolumeName,
 	})
 }
 
@@ -156,6 +159,9 @@ type netConf struct {
 	// PciAddrs in case of using sriov
 	DeviceID string `json:"deviceID"`
 	VfDriver string `json:"vf_driver"`
+	// for dpdk
+	VhostUserSocketVolumeName string `json:"vhost_user_socket_volume_name"`
+	VhostUserSocketName       string `json:"vhost_user_socket_name"`
 }
 
 func loadNetConf(bytes []byte) (*netConf, string, error) {

dist/images/Dockerfile.base-dpdk

@@ -0,0 +1,93 @@
+# syntax = docker/dockerfile:experimental
+FROM ubuntu:22.04 as ovs-builder
+
+ARG ARCH
+ARG DEBIAN_FRONTEND=noninteractive
+ENV SRC_DIR='/usr/src'
+
+RUN apt update && apt install build-essential git libnuma-dev autoconf curl \
+    python3 libmnl-dev libpcap-dev libtool libcap-ng-dev libssl-dev pkg-config \
+    python3-six libunbound-dev libunwind-dev dh-make fakeroot debhelper dh-python \
+    flake8 python3-sphinx graphviz groff wget libjemalloc-dev python3-pip -y
+
+RUN pip3 install meson ninja
+
+RUN cd /usr/src/ && \
+    wget https://fast.dpdk.org/rel/dpdk-20.11.1.tar.xz && \
+    tar xf dpdk-20.11.1.tar.xz && \
+    export DPDK_DIR=/usr/src/dpdk-stable-20.11.1 && \
+    export DPDK_BUILD=$DPDK_DIR/build && \
+    cd $DPDK_DIR && \
+    meson build && \
+    ninja -C build && \
+    ninja -C build install && \
+    ldconfig
+
+
+RUN cd /usr/src/ && \
+    git clone -b branch-2.16 --depth=1 https://github.com/openvswitch/ovs.git && \
+    cd ovs && \
+    curl -s https://github.com/kubeovn/ovs/commit/22ea22c40b46ee5adeae977ff6cfca81b3ff25d7.patch | git apply && \
+    ./boot.sh && \
+    rm -rf .git && \
+    export DPDK_DIR=/usr/src/dpdk-stable-20.11.1 && \
+    CONFIGURE_OPTS='LIBS=-ljemalloc' && \
+    if [ "$ARCH" = "amd64" ]; then CONFIGURE_OPTS='LIBS=-ljemalloc CFLAGS="-O2 -g -msse4.2 -mpopcnt"'; fi && \
+    DATAPATH_CONFIGURE_OPTS='--prefix=/usr  --with-dpdk=static' EXTRA_CONFIGURE_OPTS=$CONFIGURE_OPTS DEB_BUILD_OPTIONS='parallel=8 nocheck' fakeroot debian/rules binary
+
+RUN dpkg -i /usr/src/python3-openvswitch*.deb /usr/src/libopenvswitch*.deb
+
+RUN cd /usr/src/ && git clone -b branch-21.06 --depth=1 https://github.com/ovn-org/ovn.git && \
+    cd ovn && \
+    curl -s https://github.com/kubeovn/ovn/commit/e24734913d25c0bffdf1cfd79e14ef43d01e1019.patch | git apply && \
+    curl -s https://github.com/kubeovn/ovn/commit/8f4e4868377afb5e980856755b9f6394f8b649e2.patch | git apply && \
+    curl -s https://github.com/kubeovn/ovn/commit/23a87cabb76fbdce5092a6b3d3b56f3fa8dd61f5.patch | git apply && \
+    curl -s https://github.com/kubeovn/ovn/commit/89ca60989df4af9a96cc6024e04f99b9b77bad22.patch | git apply && \
+    curl -s https://github.com/kubeovn/ovn/commit/aeafa43fc51be8ea1c7abfbe779c69205c1c5aa4.patch | git apply && \
+    curl -s https://github.com/kubeovn/ovn/commit/71f831b9cc5a6dc923af4ca90286857e2cf8b1d3.patch | git apply && \
+    sed -i 's/OVN/ovn/g' debian/changelog && \
+    rm -rf .git && \
+    ./boot.sh && \
+    CONFIGURE_OPTS='LIBS=-ljemalloc' && \
+    if [ "$ARCH" = "amd64" ]; then CONFIGURE_OPTS='LIBS=-ljemalloc CFLAGS="-O2 -g -msse4.2 -mpopcnt"'; fi && \
+    OVSDIR=/usr/src/ovs EXTRA_CONFIGURE_OPTS=$CONFIGURE_OPTS DEB_BUILD_OPTIONS='parallel=8 nocheck' fakeroot debian/rules binary
+
+RUN mkdir /packages/ && \
+     cp /usr/src/libopenvswitch*.deb /packages && \
+     cp /usr/src/openvswitch-*.deb /packages && \
+     cp /usr/src/python3-openvswitch*.deb /packages && \
+     cp /usr/src/ovn-*.deb /packages && \
+     cd /packages && rm -f *dbg* *datapath* *docker* *vtep* *ipsec* *test* *dev*
+
+FROM ubuntu:22.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt update && apt upgrade -y && apt install ca-certificates python3 hostname libunwind8 netbase \
+        ethtool iproute2 ncat libunbound-dev procps libatomic1 kmod iptables \
+        tcpdump ipset curl uuid-runtime openssl inetutils-ping arping ndisc6 \
+        logrotate libjemalloc2 dnsutils  libnuma-dev -y --no-install-recommends && \
+        rm -rf /var/lib/apt/lists/* && \
+        cd /usr/sbin && \
+        ln -sf /usr/sbin/iptables-legacy iptables && \
+        ln -sf /usr/sbin/ip6tables-legacy ip6tables && \
+        rm -rf /etc/localtime
+
+RUN mkdir -p /var/run/openvswitch && \
+    mkdir -p /var/run/ovn && \
+    mkdir -p /etc/cni/net.d && \
+    mkdir -p /opt/cni/bin
+
+ARG ARCH
+ENV CNI_VERSION=v0.8.7
+RUN curl -sSf -L --retry 5 https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz | tar -xz -C . ./loopback ./portmap ./macvlan
+
+ENV KUBE_VERSION="v1.21.1"
+
+RUN curl -L https://dl.k8s.io/${KUBE_VERSION}/kubernetes-client-linux-${ARCH}.tar.gz | tar -xz -C . && cp ./kubernetes/client/bin/kubectl /usr/bin/kubectl \
+ && chmod +x /usr/bin/kubectl && rm -rf ./kubernetes
+
+RUN --mount=type=bind,target=/packages,from=ovs-builder,source=/packages  \
+    dpkg -i /packages/libopenvswitch*.deb && \
+    dpkg -i /packages/openvswitch-*.deb && \
+    dpkg -i /packages/python3-openvswitch*.deb &&\
+    dpkg -i --ignore-depends=openvswitch-switch,openvswitch-common /packages/ovn-*.deb

dist/images/Dockerfile.dpdk

@@ -0,0 +1,21 @@
+# syntax = docker/dockerfile:experimental
+FROM kubeovn/kube-ovn-base:v1.10.0-dpdk
+
+COPY *.sh /kube-ovn/
+COPY kubectl-ko /kube-ovn/kubectl-ko
+COPY 01-kube-ovn.conflist /kube-ovn/01-kube-ovn.conflist
+COPY logrotate/* /etc/logrotate.d/
+COPY grace_stop_ovn_controller /usr/share/ovn/scripts/grace_stop_ovn_controller
+
+WORKDIR /kube-ovn
+
+COPY kube-ovn /kube-ovn/kube-ovn
+COPY kube-ovn-cmd /kube-ovn/kube-ovn-cmd
+RUN ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-daemon && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-monitor && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-pinger && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-speaker && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-webhook && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-controller-healthcheck && \
+    ln -s /kube-ovn/kube-ovn-cmd /kube-ovn/kube-ovn-leader-checker

dist/images/cleanup.sh

@@ -49,6 +49,7 @@ done
 kubectl delete --ignore-not-found svc ovn-nb ovn-sb ovn-northd -n kube-system
 kubectl delete --ignore-not-found deploy ovn-central -n kube-system
 kubectl delete --ignore-not-found ds ovs-ovn -n kube-system
+kubectl delete --ignore-not-found ds ovs-ovn-dpdk -n kube-system
 kubectl delete --ignore-not-found secret kube-ovn-tls -n kube-system
 kubectl delete --ignore-not-found sa ovn -n kube-system
 kubectl delete --ignore-not-found clusterrole system:ovn