ci: Run the new check-system-dpdk tests as part of the ci.

This patch includes changes made earlier by David in the
ovs branch to cache the dpdk builds.

Co-authored-by: David Marchand <david.marchand@redhat.com>
Signed-off-by: David Marchand <david.marchand@redhat.com>
Signed-off-by: Eelco Chaudron <echaudro@redhat.com>
Acked-by: Ales Musil <amusil@redhat.com>
Signed-off-by: Dumitru Ceara <dceara@redhat.com>

.ci/ci.sh

@@ -16,6 +16,7 @@
 
 OVN_PATH=${OVN_PATH:-$PWD}
 OVS_PATH=${OVS_PATH:-$OVN_PATH/ovs}
+DPDK_PATH=${DPDK_PATH:-$OVN_PATH/dpdk-dir}
 CONTAINER_CMD=${CONTAINER_CMD:-podman}
 CONTAINER_WORKSPACE="/workspace"
 CONTAINER_WORKDIR="/workspace/ovn-tmp"
@@ -80,6 +81,10 @@ function copy_sources_to_workdir() {
         && \
         cp -a $CONTAINER_WORKSPACE/ovs/. $CONTAINER_WORKDIR/ovs \
         && \
+        rm -rf $CONTAINER_WORKDIR/dpdk-dir \
+        && \
+        cp -a $CONTAINER_WORKSPACE/dpdk-dir/. $CONTAINER_WORKDIR/dpdk-dir \
+        && \
         git config --global --add safe.directory $CONTAINER_WORKDIR
     "
 }
@@ -95,7 +100,7 @@ function run_tests() {
         cd $CONTAINER_WORKDIR \
         && \
         ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \
-        TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS \
+        TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \
         ./.ci/linux-build.sh
     "
 }
@@ -148,12 +153,17 @@ if [ "$ARCH" = "aarch64" ]; then
     ASAN_OPTIONS="detect_leaks=0"
 fi
 
+if [ -z "$DPDK" ]; then
+   mkdir -p "$DPDK_PATH"
+fi
+
 CONTAINER_ID="$($CONTAINER_CMD run --privileged -d \
     --pids-limit=-1 \
     --env ASAN_OPTIONS=$ASAN_OPTIONS \
     -v /lib/modules/$(uname -r):/lib/modules/$(uname -r):ro \
     -v $OVN_PATH:$CONTAINER_WORKSPACE/ovn:Z \
     -v $OVS_PATH:$CONTAINER_WORKSPACE/ovs:Z \
+    -v $DPDK_PATH:$CONTAINER_WORKSPACE/dpdk-dir:Z \
     $IMAGE_NAME)"
 trap remove_container EXIT
 

.ci/dpdk-build.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -o errexit
+set -x
+
+function build_dpdk()
+{
+    local VERSION_FILE="dpdk-dir/cached-version"
+    local DPDK_VER=$1
+    local DPDK_OPTS=""
+
+    rm -rf dpdk-dir
+
+    if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
+        git clone --single-branch $DPDK_GIT dpdk-dir -b "${DPDK_VER##refs/*/}"
+        pushd dpdk-dir
+        git log -1 --oneline
+    else
+        wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz
+        tar xvf dpdk-$1.tar.xz > /dev/null
+        DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/")
+        mv ${DIR_NAME} dpdk-dir
+        pushd dpdk-dir
+    fi
+
+    # Switching to 'default' machine to make dpdk-dir cache usable on
+    # different CPUs. We can't be sure that all CI machines are exactly same.
+    DPDK_OPTS="$DPDK_OPTS -Dmachine=default"
+
+    # Disable building DPDK unit tests. Not needed for OVS build or tests.
+    DPDK_OPTS="$DPDK_OPTS -Dtests=false"
+
+    # Disable DPDK developer mode, this results in less build checks and less
+    # meson verbose outputs.
+    DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled"
+
+    # OVS compilation and the "ovn-system-dpdk" unit tests (run in the CI)
+    # only depend on virtio/tap drivers.
+    # We can disable all remaining drivers to save compilation time.
+    DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio"
+
+    # Install DPDK using prefix.
+    DPDK_OPTS="$DPDK_OPTS --prefix=$(pwd)/build"
+
+    meson $DPDK_OPTS build
+    ninja -C build
+    ninja -C build install
+
+    echo "Installed DPDK in $(pwd)"
+    popd
+    echo "${DPDK_VER}" > ${VERSION_FILE}
+}
+
+build_dpdk $DPDK_VER

.ci/dpdk-prepare.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -ev
+
+# Installing wheel separately because it may be needed to build some
+# of the packages during dependency backtracking and pip >= 22.0 will
+# abort backtracking on build failures:
+#     https://github.com/pypa/pip/issues/10655
+pip3 install --disable-pip-version-check --user wheel
+pip3 install --disable-pip-version-check --user pyelftools
+pip3 install --user  'meson==0.53.2'

.ci/linux-build.sh

@@ -10,13 +10,54 @@ OVN_CFLAGS=""
 OPTS="$OPTS --enable-Werror"
 JOBS=${JOBS:-"-j4"}
 
+function install_dpdk()
+{
+    local VERSION_FILE="dpdk-dir/cached-version"
+    local DPDK_LIB=$(pwd)/dpdk-dir/build/lib/x86_64-linux-gnu
+
+    # Export the following path for pkg-config to find the .pc file.
+    export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH
+
+    if [ ! -f "${VERSION_FILE}" ]; then
+        echo "Could not find DPDK in $(pwd)/dpdk-dir"
+        return 1
+    fi
+
+    # As we build inside a container we need to update the prefix.
+    sed -i -E "s|^prefix=.*|prefix=$(pwd)/dpdk-dir/build|" \
+        "$DPDK_LIB/pkgconfig/libdpdk-libs.pc"
+
+    # Update the library paths.
+    sudo ldconfig
+    echo "Found cached DPDK $(cat ${VERSION_FILE}) build in $(pwd)/dpdk-dir"
+}
+
 function configure_ovs()
 {
+    if [ "$DPDK" ]; then
+        # When DPDK is enabled, we need to build OVS twice. Once to have
+        # ovs-vswitchd with DPDK. But OVN does not like the OVS libraries to
+        # be compiled with DPDK enabled, hence we need a final clean build
+        # with this disabled.
+        install_dpdk
+
+        pushd ovs
+        ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" --with-dpdk=static \
+            $* || { cat config.log; exit 1; }
+        make $JOBS || { cat config.log; exit 1; }
+        cp vswitchd/ovs-vswitchd vswitchd/ovs-vswitchd_with_dpdk
+        popd
+    fi
+
     pushd ovs
     ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" $* || \
-    { cat config.log; exit 1; }
+        { cat config.log; exit 1; }
     make $JOBS || { cat config.log; exit 1; }
     popd
+
+    if [ "$DPDK" ]; then
+        cp ovs/vswitchd/ovs-vswitchd_with_dpdk ovs/vswitchd/ovs-vswitchd
+    fi
 }
 
 function configure_ovn()
@@ -104,6 +145,12 @@ if [ "$TESTSUITE" ]; then
         execute_system_tests "check-system-userspace" \
             "system-userspace-testsuite.log"
         ;;
+
+        "system-test-dpdk")
+        # The dpdk tests need huge page memory, so reserve some 2M pages.
+        sudo bash -c "echo 2048 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
+        execute_system_tests "check-system-dpdk" "system-dpdk-testsuite.log"
+        ;;
     esac
 else
     configure_ovn $OPTS

.github/workflows/test.yml

@@ -12,11 +12,81 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  build-dpdk:
+    env:
+      dependencies: gcc libnuma-dev ninja-build
+      CC: gcc
+      DPDK_GIT: https://dpdk.org/git/dpdk-stable
+      DPDK_VER: 22.11.1
+    name: dpdk gcc
+    outputs:
+      dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }}
+    runs-on: ubuntu-20.04
+    timeout-minutes: 30
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v3
+
+    - name: update PATH
+      run: |
+        echo "$HOME/bin"        >> $GITHUB_PATH
+        echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+    - name: create ci signature file for the dpdk cache key
+      # This will collect most of DPDK related lines, so hash will be different
+      # if something changed in a way we're building DPDK including DPDK_VER.
+      # This also allows us to use cache from any branch as long as version
+      # and a way we're building DPDK stays the same.
+      run: |
+        grep -irE 'RTE_|DPDK|meson|ninja' .ci/dpdk-* > dpdk-ci-signature
+        grep -rwE 'DPDK_GIT|DPDK_VER' .github/ >> dpdk-ci-signature
+        if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
+            git ls-remote --heads $DPDK_GIT $DPDK_VER >> dpdk-ci-signature
+        fi
+        cat dpdk-ci-signature
+
+    - name: generate ci DPDK key
+      id: gen_dpdk_key
+      env:
+        ci_key: ${{ hashFiles('dpdk-ci-signature') }}
+      run: echo 'key=dpdk-${{ env.ci_key }}' >> $GITHUB_OUTPUT
+
+    - name: cache
+      id: dpdk_cache
+      uses: actions/cache@v3
+      with:
+        path: dpdk-dir
+        key: ${{ steps.gen_dpdk_key.outputs.key }}
+
+    - name: set up python
+      if: steps.dpdk_cache.outputs.cache-hit != 'true'
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.9'
+
+    - name: update APT cache
+      if: steps.dpdk_cache.outputs.cache-hit != 'true'
+      run: sudo apt update || true
+    - name: install common dependencies
+      if: steps.dpdk_cache.outputs.cache-hit != 'true'
+      run: sudo apt install -y ${{ env.dependencies }}
+
+    - name: prepare
+      if: steps.dpdk_cache.outputs.cache-hit != 'true'
+      run: ./.ci/dpdk-prepare.sh
+
+    - name: build
+      if: steps.dpdk_cache.outputs.cache-hit != 'true'
+      run: ./.ci/dpdk-build.sh
+
   build-linux:
+    needs: build-dpdk
     env:
       IMAGE_NAME:  ghcr.io/ovn-org/ovn-tests:ubuntu
       ARCH:        ${{ matrix.cfg.arch }}
       CC:          ${{ matrix.cfg.compiler }}
+      DPDK:        ${{ matrix.cfg.dpdk }}
       LIBS:        ${{ matrix.cfg.libs }}
       OPTS:        ${{ matrix.cfg.opts }}
       TESTSUITE:   ${{ matrix.cfg.testsuite }}
@@ -43,6 +113,9 @@ jobs:
         - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: "-500" }
         - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: "501-1000" }
         - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: "1001-" }
+        - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, test_range: "-100" }
+        - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, test_range: "101-200" }
+        - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, test_range: "201-" }
         - { compiler: gcc, testsuite: system-test-userspace, test_range: "-100" }
         - { compiler: gcc, testsuite: system-test-userspace, test_range: "101-200" }
         - { compiler: gcc, testsuite: system-test-userspace, test_range: "201-" }
@@ -84,6 +157,13 @@ jobs:
             sort -V | tail -1)
       working-directory: ovs
 
+    - name: cache
+      if: matrix.cfg.dpdk != ''
+      uses: actions/cache@v3
+      with:
+        path: dpdk-dir
+        key: ${{ needs.build-dpdk.outputs.dpdk_key }}
+
     - name: build
       if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }}
       run: sudo -E ./.ci/ci.sh --archive-logs

Makefile.am

@@ -87,6 +87,8 @@ EXTRA_DIST = \
 	NOTICE \
 	.cirrus.yml \
 	.ci/ci.sh \
+	.ci/dpdk-build.sh \
+	.ci/dpdk-prepare.sh \
 	.ci/linux-build.sh \
 	.ci/osx-build.sh \
 	.ci/osx-prepare.sh \

tests/system-ovn.at

@@ -4267,7 +4267,7 @@ done
 # Enable IGMP snooping on sw1.
 ovn-nbctl set Logical_Switch sw1 other_config:mcast_querier="false"
 ovn-nbctl set Logical_Switch sw1 other_config:mcast_snoop="true"
-
+ovn-nbctl --wait=hv sync
 
 group_v4="239.0.1.68"
 # Inject IGMP Join for v4 group on sw1-p1.