From a6fdafa03bbdf7ff2590a20e078b1a1a37ee8c4b Mon Sep 17 00:00:00 2001 From: Kernel Patches Daemon Date: Wed, 31 Aug 2022 10:43:38 -0700 Subject: [PATCH 1/2] adding ci files --- .github/workflows/test.yml | 165 ++++++++++++++++++++++++ README | 18 --- travis-ci/vmtest/configs/DENYLIST | 6 + travis-ci/vmtest/configs/DENYLIST.s390x | 2 + travis-ci/vmtest/helpers.sh | 36 ++++++ travis-ci/vmtest/run_selftests.sh | 93 +++++++++++++ 6 files changed, 302 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 travis-ci/vmtest/configs/DENYLIST create mode 100644 travis-ci/vmtest/configs/DENYLIST.s390x create mode 100755 travis-ci/vmtest/helpers.sh create mode 100755 travis-ci/vmtest/run_selftests.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000000..43559c261bb35 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,165 @@ +name: bpf-ci + +on: + pull_request: + +concurrency: + group: ci-test-${{ github.head_ref }} + cancel-in-progress: true + +jobs: + llvm-toolchain: + runs-on: ubuntu-latest + outputs: + llvm: ${{ steps.llvm-toolchain-impl.outputs.version }} + steps: + - id: llvm-version + uses: libbpf/ci/get-llvm-version@master + - id: llvm-toolchain-impl + shell: bash + run: echo "::set-output name=version::llvm-${{ steps.llvm-version.outputs.version }}" + set-matrix: + needs: llvm-toolchain + runs-on: ubuntu-latest + outputs: + build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }} + test-matrix: ${{ steps.set-matrix-impl.outputs.test_matrix }} + steps: + - id: set-matrix-impl + shell: python3 -I {0} + run: | + from json import dumps + + matrix = [ + {"kernel": "LATEST", "runs_on": ["ubuntu-latest", "self-hosted"], "arch": "x86_64", "toolchain": "gcc"}, + {"kernel": "LATEST", "runs_on": ["ubuntu-latest", "self-hosted"], "arch": "x86_64", "toolchain": "${{ needs.llvm-toolchain.outputs.llvm }}"}, + {"kernel": "LATEST", "runs_on": ["z15", "self-hosted"], "arch": "s390x", "toolchain": "gcc"}, + ] + + if "${{ github.repository_owner }}" != "kernel-patches": + # Outside of kernel-patches, remove the self-hosted label and skip + # any testing on s390x, as no suitable runners will be available. + for idx in range(len(matrix) - 1, -1, -1): + if "z15" in matrix[idx]["runs_on"]: + del matrix[idx] + else: + matrix[idx]["runs_on"].remove("self-hosted") + + build_matrix = {"include": matrix} + print(f"::set-output name=build_matrix::{dumps(build_matrix)}") + + tests = ["test_progs", "test_progs_no_alu32", "test_maps", "test_verifier"] + test_matrix = {"include": [{**config, **{"test": test}} + for config in matrix + for test in tests]} + print(f"::set-output name=test_matrix::{dumps(test_matrix)}") + build: + name: build for ${{ matrix.arch }} with ${{ matrix.toolchain }} + needs: set-matrix + runs-on: ${{ matrix.runs_on }} + timeout-minutes: 100 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }} + env: + KERNEL: ${{ matrix.kernel }} + REPO_ROOT: ${{ github.workspace }} + REPO_PATH: "" + steps: + - uses: actions/checkout@v2 + - if: ${{ github.repository == 'kernel-patches/vmtest' }} + name: Download bpf-next tree + uses: libbpf/ci/get-linux-source@master + with: + dest: '.kernel' + - if: ${{ github.repository == 'kernel-patches/vmtest' }} + name: Move linux source in place + shell: bash + run: | + rm -rf .kernel/.git + cp -rf .kernel/. . + rm -rf .kernel + - uses: libbpf/ci/patch-kernel@master + with: + patches-root: '${{ github.workspace }}/travis-ci/diffs' + repo-root: '${{ github.workspace }}' + - name: Setup build environment + uses: libbpf/ci/setup-build-env@master + - name: Build kernel image + uses: libbpf/ci/build-linux@master + with: + arch: ${{ matrix.arch }} + toolchain: ${{ matrix.toolchain }} + - name: Build selftests + uses: libbpf/ci/build-selftests@master + with: + vmlinux_btf: ${{ github.workspace }}/vmlinux + toolchain: ${{ matrix.toolchain }} + - name: Build samples + uses: libbpf/ci/build-samples@master + with: + vmlinux_btf: ${{ github.workspace }}/vmlinux + toolchain: ${{ matrix.toolchain }} + - name: Tar artifacts + run: | + file_list="" + if [ "${{ github.repository }}" == "kernel-patches/vmtest" ]; then + # Package up a bunch of additional infrastructure to support running + # 'make kernelrelease' and bpf tool checks later on. + file_list="$(find . -iname Makefile | xargs) \ + scripts/ \ + tools/testing/selftests/bpf/ \ + tools/include/ \ + tools/bpf/bpftool/"; + fi + + tar -czf vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.gz \ + .config \ + arch/*/boot/bzImage \ + include/config/auto.conf \ + include/generated/autoconf.h \ + ${file_list} \ + --exclude '*.h' \ + selftests/bpf/ \ + vmlinux + - uses: actions/upload-artifact@v3 + with: + name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }} + if-no-files-found: error + path: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.gz + test: + name: ${{ matrix.test }} on ${{ matrix.arch }} with ${{ matrix.toolchain }} + needs: [set-matrix, build] + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.set-matrix.outputs.test-matrix) }} + runs-on: ${{ matrix.runs_on }} + timeout-minutes: 100 + env: + KERNEL: ${{ matrix.kernel }} + REPO_ROOT: ${{ github.workspace }} + REPO_PATH: "" + steps: + - uses: actions/checkout@main + - uses: actions/download-artifact@v3 + with: + name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }} + path: . + - name: Untar artifacts + run: tar -xzf vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.gz + - name: Prepare rootfs + uses: libbpf/ci/prepare-rootfs@master + with: + project-name: 'libbpf' + arch: ${{ matrix.arch }} + kernel: ${{ matrix.kernel }} + kernel-root: '.' + image-output: '/tmp/root.img' + test: ${{ matrix.test }} + - name: Run selftests + uses: libbpf/ci/run-qemu@master + with: + arch: ${{ matrix.arch}} + img: '/tmp/root.img' + vmlinuz: '${{ github.workspace }}/vmlinuz' + kernel-root: '.' diff --git a/README b/README index 669ac7c322927..e69de29bb2d1d 100644 --- a/README +++ b/README @@ -1,18 +0,0 @@ -Linux kernel -============ - -There are several guides for kernel developers and users. These guides can -be rendered in a number of formats, like HTML and PDF. Please read -Documentation/admin-guide/README.rst first. - -In order to build the documentation, use ``make htmldocs`` or -``make pdfdocs``. The formatted documentation can also be read online at: - - https://www.kernel.org/doc/html/latest/ - -There are various text files in the Documentation/ subdirectory, -several of them using the Restructured Text markup notation. - -Please read the Documentation/process/changes.rst file, as it contains the -requirements for building and running the kernel, and information about -the problems which may result by upgrading your kernel. diff --git a/travis-ci/vmtest/configs/DENYLIST b/travis-ci/vmtest/configs/DENYLIST new file mode 100644 index 0000000000000..d12cf9fae5eee --- /dev/null +++ b/travis-ci/vmtest/configs/DENYLIST @@ -0,0 +1,6 @@ +# TEMPORARY +btf_dump/btf_dump: syntax +kprobe_multi_test/bench_attach +core_reloc/enum64val +core_reloc/size___diff_sz +core_reloc/type_based___diff_sz diff --git a/travis-ci/vmtest/configs/DENYLIST.s390x b/travis-ci/vmtest/configs/DENYLIST.s390x new file mode 100644 index 0000000000000..66d628bd6d206 --- /dev/null +++ b/travis-ci/vmtest/configs/DENYLIST.s390x @@ -0,0 +1,2 @@ +tc_redirect/tc_redirect_dtime # very flaky +lru_bug # not yet in bpf-next denylist diff --git a/travis-ci/vmtest/helpers.sh b/travis-ci/vmtest/helpers.sh new file mode 100755 index 0000000000000..3b2cda0153b71 --- /dev/null +++ b/travis-ci/vmtest/helpers.sh @@ -0,0 +1,36 @@ +# $1 - start or end +# $2 - fold identifier, no spaces +# $3 - fold section description +foldable() { + local YELLOW='\033[1;33m' + local NOCOLOR='\033[0m' + if [ $1 = "start" ]; then + line="::group::$2" + if [ ! -z "${3:-}" ]; then + line="$line - ${YELLOW}$3${NOCOLOR}" + fi + else + line="::endgroup::" + fi + echo -e "$line" +} + +__print() { + local TITLE="" + if [[ -n $2 ]]; then + TITLE=" title=$2" + fi + echo "::$1${TITLE}::$3" +} + +# $1 - title +# $2 - message +print_error() { + __print error $1 $2 +} + +# $1 - title +# $2 - message +print_notice() { + __print notice $1 $2 +} diff --git a/travis-ci/vmtest/run_selftests.sh b/travis-ci/vmtest/run_selftests.sh new file mode 100755 index 0000000000000..416960dd029d1 --- /dev/null +++ b/travis-ci/vmtest/run_selftests.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -euo pipefail + +source $(cd $(dirname $0) && pwd)/helpers.sh + +ARCH=$(uname -m) + +STATUS_FILE=/exitstatus + +read_lists() { + (for path in "$@"; do + if [[ -s "$path" ]]; then + cat "$path" + fi; + done) | cut -d'#' -f1 | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' | tr -s '\n' ',' +} + +TEST_PROGS_ARGS="" +# Disabled due to issue +# if [[ "$(nproc)" -gt 2 ]]; then +# TEST_PROGS_ARGS="-j" +# fi + +test_progs() { + foldable start test_progs "Testing test_progs" + # "&& true" does not change the return code (it is not executed + # if the Python script fails), but it prevents exiting on a + # failure due to the "set -e". + ./test_progs ${DENYLIST:+-d"$DENYLIST"} ${ALLOWLIST:+-a"$ALLOWLIST"} ${TEST_PROGS_ARGS} && true + echo "test_progs:$?" >>"${STATUS_FILE}" + foldable end test_progs +} + +test_progs_no_alu32() { + foldable start test_progs-no_alu32 "Testing test_progs-no_alu32" + ./test_progs-no_alu32 ${DENYLIST:+-d"$DENYLIST"} ${ALLOWLIST:+-a"$ALLOWLIST"} ${TEST_PROGS_ARGS} && true + echo "test_progs-no_alu32:$?" >>"${STATUS_FILE}" + foldable end test_progs-no_alu32 +} + +test_maps() { + foldable start test_maps "Testing test_maps" + taskset 0xF ./test_maps && true + echo "test_maps:$?" >>"${STATUS_FILE}" + foldable end test_maps +} + +test_verifier() { + foldable start test_verifier "Testing test_verifier" + ./test_verifier && true + echo "test_verifier:$?" >>"${STATUS_FILE}" + foldable end test_verifier +} + +foldable end vm_init + +foldable start kernel_config "Kconfig" + +zcat /proc/config.gz + +foldable end kernel_config + +configs_path=${PROJECT_NAME}/selftests/bpf +local_configs_path=${PROJECT_NAME}/vmtest/configs +DENYLIST=$(read_lists \ + "$configs_path/DENYLIST" \ + "$configs_path/DENYLIST.${ARCH}" \ + "$local_configs_path/DENYLIST" \ + "$local_configs_path/DENYLIST.${ARCH}" \ +) +ALLOWLIST=$(read_lists \ + "$configs_path/ALLOWLIST" \ + "$configs_path/ALLOWLIST.${ARCH}" \ + "$local_configs_path/ALLOWLIST" \ + "$local_configs_path/ALLOWLIST.${ARCH}" \ +) + +echo "DENYLIST: ${DENYLIST}" +echo "ALLOWLIST: ${ALLOWLIST}" + +cd ${PROJECT_NAME}/selftests/bpf + +if [ $# -eq 0 ]; then + test_progs + test_progs_no_alu32 + test_maps + test_verifier +else + for test_name in "$@"; do + "${test_name}" + done +fi From c2760106e5cee644958ef96b79c77b51350d43da Mon Sep 17 00:00:00 2001 From: James Hilliard Date: Mon, 29 Aug 2022 15:05:46 -0600 Subject: [PATCH 2/2] libbpf: add GCC support for bpf_tail_call_static The bpf_tail_call_static function is currently not defined unless using clang >= 8. To support bpf_tail_call_static on GCC we can check if __clang__ is not defined to enable bpf_tail_call_static. We need to use GCC assembly syntax when the compiler does not define __clang__ as LLVM inline assembly is not fully compatible with GCC. Signed-off-by: James Hilliard --- tools/lib/bpf/bpf_helpers.h | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 7349b16b8e2f0..867b734839ddf 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -131,7 +131,7 @@ /* * Helper function to perform a tail call with a constant/immediate map slot. */ -#if __clang_major__ >= 8 && defined(__bpf__) +#if (!defined(__clang__) || __clang_major__ >= 8) && defined(__bpf__) static __always_inline void bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) { @@ -139,8 +139,8 @@ bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) __bpf_unreachable(); /* - * Provide a hard guarantee that LLVM won't optimize setting r2 (map - * pointer) and r3 (constant map index) from _different paths_ ending + * Provide a hard guarantee that the compiler won't optimize setting r2 + * (map pointer) and r3 (constant map index) from _different paths_ ending * up at the _same_ call insn as otherwise we won't be able to use the * jmpq/nopl retpoline-free patching by the x86-64 JIT in the kernel * given they mismatch. See also d2e4c1e6c294 ("bpf: Constant map key @@ -148,12 +148,19 @@ bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) * * Note on clobber list: we need to stay in-line with BPF calling * convention, so even if we don't end up using r0, r4, r5, we need - * to mark them as clobber so that LLVM doesn't end up using them - * before / after the call. + * to mark them as clobber so that the compiler doesn't end up using + * them before / after the call. */ - asm volatile("r1 = %[ctx]\n\t" + asm volatile( +#ifdef __clang__ + "r1 = %[ctx]\n\t" "r2 = %[map]\n\t" "r3 = %[slot]\n\t" +#else + "mov %%r1,%[ctx]\n\t" + "mov %%r2,%[map]\n\t" + "mov %%r3,%[slot]\n\t" +#endif "call 12" :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot) : "r0", "r1", "r2", "r3", "r4", "r5");