kernel-patches · kernel-patches-bot · Oct 26, 2022
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -0,0 +1,177 @@
+name: bpf-ci
+
+on:
+  pull_request:
+
+concurrency:
+  group: ci-test-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  llvm-toolchain:
+    runs-on: ubuntu-latest
+    outputs:
+      llvm: ${{ steps.llvm-toolchain-impl.outputs.version }}
+    steps:
+      - id: llvm-version
+        uses: libbpf/ci/get-llvm-version@master
+      - id: llvm-toolchain-impl
+        shell: bash
+        run: echo "version=llvm-${{ steps.llvm-version.outputs.version }}" >> $GITHUB_OUTPUT
+  set-matrix:
+    needs: llvm-toolchain
+    runs-on: ubuntu-latest
+    outputs:
+      build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }}
+      test-matrix: ${{ steps.set-matrix-impl.outputs.test_matrix }}
+    steps:
+      - id: set-matrix-impl
+        shell: python3 -I {0}
+        run: |
+          from json import dumps
+          import os
+
+          def set_output(name, value):
+            """Write an output variable to the GitHub output file."""
+            with open(os.getenv("GITHUB_OUTPUT"), "a") as f:
+              f.write(f"{name}={value}\n")
+
+          matrix = [
+            {"kernel": "LATEST", "runs_on": ["ubuntu-latest", "self-hosted"], "arch": "x86_64", "toolchain": "gcc"},
+            {"kernel": "LATEST", "runs_on": ["ubuntu-latest", "self-hosted"], "arch": "x86_64", "toolchain": "${{ needs.llvm-toolchain.outputs.llvm }}"},
+            {"kernel": "LATEST", "runs_on": ["z15", "self-hosted"], "arch": "s390x", "toolchain": "gcc"},
+          ]
+          self_hosted_repos = [
+            "kernel-patches/bpf",
+            "kernel-patches/vmtest",
+          ]
+
+          # Only a few repository within "kernel-patches" use self-hosted runners.
+          if "${{ github.repository_owner }}" != "kernel-patches" or "${{ github.repository }}" not in self_hosted_repos:
+            # Outside of those repositories, remove the self-hosted label and skip
+            # any testing on s390x, as no suitable runners will be available.
+            for idx in range(len(matrix) - 1, -1, -1):
+              if "z15" in matrix[idx]["runs_on"]:
+                del matrix[idx]
+              else:
+                matrix[idx]["runs_on"].remove("self-hosted")
+
+          build_matrix = {"include": matrix}
+          set_output("build_matrix", dumps(build_matrix))
+
+          tests = ["test_progs", "test_progs_no_alu32", "test_maps", "test_verifier"]
+          test_matrix = {"include": [{**config, **{"test": test}}
+                                      for config in matrix
+                                        for test in tests]}
+          set_output("test_matrix", dumps(test_matrix))
+  build:
+    name: build for ${{ matrix.arch }} with ${{ matrix.toolchain }}
+    needs: set-matrix
+    runs-on: ${{ matrix.runs_on }}
+    timeout-minutes: 100
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.set-matrix.outputs.build-matrix) }}
+    env:
+      KERNEL: ${{ matrix.kernel }}
+      REPO_ROOT: ${{ github.workspace }}
+      REPO_PATH: ""
+    steps:
+      - uses: actions/checkout@v3
+      - if: ${{ github.repository == 'kernel-patches/vmtest' }}
+        name: Download bpf-next tree
+        uses: libbpf/ci/get-linux-source@master
+        with:
+          dest: '.kernel'
+      - if: ${{ github.repository == 'kernel-patches/vmtest' }}
+        name: Move linux source in place
+        shell: bash
+        run: |
+          rm -rf .kernel/.git
+          cp -rf .kernel/. .
+          rm -rf .kernel
+      - uses: libbpf/ci/patch-kernel@master
+        with:
+          patches-root: '${{ github.workspace }}/ci/diffs'
+          repo-root: '${{ github.workspace }}'
+      - name: Setup build environment
+        uses: libbpf/ci/setup-build-env@master
+      - name: Build kernel image
+        uses: libbpf/ci/build-linux@master
+        with:
+          arch: ${{ matrix.arch }}
+          toolchain: ${{ matrix.toolchain }}
+      - name: Build selftests
+        uses: libbpf/ci/build-selftests@master
+        with:
+          vmlinux_btf: ${{ github.workspace }}/vmlinux
+          toolchain: ${{ matrix.toolchain }}
+      - name: Build samples
+        uses: libbpf/ci/build-samples@master
+        with:
+          vmlinux_btf: ${{ github.workspace }}/vmlinux
+          toolchain: ${{ matrix.toolchain }}
+      - name: Tar artifacts
+        run: |
+          file_list=""
+          if [ "${{ github.repository }}" == "kernel-patches/vmtest" ]; then
+            # Package up a bunch of additional infrastructure to support running
+            # 'make kernelrelease' and bpf tool checks later on.
+            file_list="$(find . -iname Makefile | xargs) \
+              scripts/ \
+              tools/testing/selftests/bpf/ \
+              tools/include/ \
+              tools/bpf/bpftool/";
+          fi
+          # zstd is installed by default in the runner images.
+          tar -cf - \
+            .config \
+            arch/*/boot/bzImage \
+            include/config/auto.conf \
+            include/generated/autoconf.h \
+            ${file_list} \
+            --exclude '*.h' \
+            selftests/bpf/ \
+            vmlinux | zstd -T0 -19 -o vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.zst
+      - uses: actions/upload-artifact@v3
+        with:
+          name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}
+          if-no-files-found: error
+          path: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.zst
+  test:
+    name: ${{ matrix.test }} on ${{ matrix.arch }} with ${{ matrix.toolchain }}
+    needs: [set-matrix, build]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.set-matrix.outputs.test-matrix) }}
+    runs-on: ${{ matrix.runs_on }}
+    timeout-minutes: 100
+    env:
+      KERNEL: ${{ matrix.kernel }}
+      REPO_ROOT: ${{ github.workspace }}
+      REPO_PATH: ""
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/download-artifact@v3
+        with:
+          name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}
+          path: .
+      - name: Untar artifacts
+        # zstd is installed by default in the runner images.
+        run: zstd -d -T0  vmlinux-${{ matrix.arch }}-${{ matrix.toolchain }}.tar.zst --stdout | tar -xf -
+      - name: Prepare rootfs
+        uses: libbpf/ci/prepare-rootfs@master
+        with:
+          project-name: 'libbpf'
+          arch: ${{ matrix.arch }}
+          kernel: ${{ matrix.kernel }}
+          kernel-root: '.'
+          image-output: '/tmp/root.img'
+          test: ${{ matrix.test }}
+      - name: Run selftests
+        uses: libbpf/ci/run-qemu@master
+        with:
+          arch: ${{ matrix.arch}}
+          img: '/tmp/root.img'
+          vmlinuz: '${{ github.workspace }}/vmlinuz'
+          kernel-root: '.'
diff --git a/README b/README
@@ -1,18 +0,0 @@
-Linux kernel
-============
-
-There are several guides for kernel developers and users. These guides can
-be rendered in a number of formats, like HTML and PDF. Please read
-Documentation/admin-guide/README.rst first.
-
-In order to build the documentation, use ``make htmldocs`` or
-``make pdfdocs``.  The formatted documentation can also be read online at:
-
-    https://www.kernel.org/doc/html/latest/
-
-There are various text files in the Documentation/ subdirectory,
-several of them using the Restructured Text markup notation.
-
-Please read the Documentation/process/changes.rst file, as it contains the
-requirements for building and running the kernel, and information about
-the problems which may result by upgrading your kernel.

diff --git a/ci/diffs/.do_not_use_dot_patch_here b/ci/diffs/.do_not_use_dot_patch_here
diff --git a/ci/diffs/0001-bpf-Fix-sample_flags-for-bpf_perf_event_output.diff b/ci/diffs/0001-bpf-Fix-sample_flags-for-bpf_perf_event_output.diff
@@ -0,0 +1,47 @@
+From 01cc82fac426032c414258f2c322f63e75cf04c0 Mon Sep 17 00:00:00 2001
+From: Sumanth Korikkar <sumanthk@linux.ibm.com>
+Date: Fri, 7 Oct 2022 10:13:27 +0200
+Subject: [PATCH] bpf: Fix sample_flags for bpf_perf_event_output
+
+* Raw data is also filled by bpf_perf_event_output.
+* Add sample_flags to indicate raw data.
+* This eliminates the segfaults as shown below:
+  Run ./samples/bpf/trace_output
+  BUG pid 9 cookie 1001000000004 sized 4
+  BUG pid 9 cookie 1001000000004 sized 4
+  BUG pid 9 cookie 1001000000004 sized 4
+  Segmentation fault (core dumped)
+
+Fixes: 838d9bb62d13 ("perf: Use sample_flags for raw_data")
+Signed-off-by: Sumanth Korikkar <sumanthk@linux.ibm.com>
+Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
+Acked-by: Namhyung Kim <namhyung@kernel.org>
+Acked-by: Jiri Olsa <jolsa@kernel.org>
+Link: https://lore.kernel.org/bpf/20221007081327.1047552-1-sumanthk@linux.ibm.com
+---
+ kernel/trace/bpf_trace.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c
+index 688552df95ca..596335fe62e2 100644
+--- a/kernel/trace/bpf_trace.c
++++ b/kernel/trace/bpf_trace.c
+@@ -687,6 +687,7 @@ BPF_CALL_5(bpf_perf_event_output, struct pt_regs *, regs, struct bpf_map *, map,
+
+ 	perf_sample_data_init(sd, 0, 0);
+ 	sd->raw = &raw;
++	sd->sample_flags |= PERF_SAMPLE_RAW;
+
+ 	err = __bpf_perf_event_output(regs, map, flags, sd);
+
+@@ -745,6 +746,7 @@ u64 bpf_event_output(struct bpf_map *map, u64 flags, void *meta, u64 meta_size,
+ 	perf_fetch_caller_regs(regs);
+ 	perf_sample_data_init(sd, 0, 0);
+ 	sd->raw = &raw;
++	sd->sample_flags |= PERF_SAMPLE_RAW;
+
+ 	ret = __bpf_perf_event_output(regs, map, flags, sd);
+ out:
+-- 
+2.30.2
+
diff --git a/ci/diffs/0001-selftests-bpf-S-iptables-iptables-legacy-in-the-bpf_.diff b/ci/diffs/0001-selftests-bpf-S-iptables-iptables-legacy-in-the-bpf_.diff
@@ -0,0 +1,77 @@
+From de9c8d848d90cf2e53aced50b350827442ca5a4f Mon Sep 17 00:00:00 2001
+From: Martin KaFai Lau <martin.lau@kernel.org>
+Date: Wed, 12 Oct 2022 15:12:35 -0700
+Subject: [PATCH] selftests/bpf: S/iptables/iptables-legacy/ in the bpf_nf and
+ xdp_synproxy test
+
+The recent vm image in CI has reported error in selftests that use
+the iptables command.  Manu Bretelle has pointed out the difference
+in the recent vm image that the iptables is sym-linked to the iptables-nft.
+With this knowledge,  I can also reproduce the CI error by manually running
+with the 'iptables-nft'.
+
+This patch is to replace the iptables command with iptables-legacy
+to unblock the CI tests.
+
+Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
+Signed-off-by: Andrii Nakryiko <andrii@kernel.org>
+Acked-by: David Vernet <void@manifault.com>
+Link: https://lore.kernel.org/bpf/20221012221235.3529719-1-martin.lau@linux.dev
+---
+ tools/testing/selftests/bpf/prog_tests/bpf_nf.c       | 6 +++---
+ tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c | 6 +++---
+ 2 files changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
+index 8a838ea8bdf3..c8ba4009e4ab 100644
+--- a/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
++++ b/tools/testing/selftests/bpf/prog_tests/bpf_nf.c
+@@ -49,14 +49,14 @@ static int connect_to_server(int srv_fd)
+
+ static void test_bpf_nf_ct(int mode)
+ {
+-	const char *iptables = "iptables -t raw %s PREROUTING -j CONNMARK --set-mark 42/0";
++	const char *iptables = "iptables-legacy -t raw %s PREROUTING -j CONNMARK --set-mark 42/0";
+ 	int srv_fd = -1, client_fd = -1, srv_client_fd = -1;
+ 	struct sockaddr_in peer_addr = {};
+ 	struct test_bpf_nf *skel;
+ 	int prog_fd, err;
+ 	socklen_t len;
+ 	u16 srv_port;
+-	char cmd[64];
++	char cmd[128];
+ 	LIBBPF_OPTS(bpf_test_run_opts, topts,
+ 		.data_in = &pkt_v4,
+ 		.data_size_in = sizeof(pkt_v4),
+@@ -69,7 +69,7 @@ static void test_bpf_nf_ct(int mode)
+
+ 	/* Enable connection tracking */
+ 	snprintf(cmd, sizeof(cmd), iptables, "-A");
+-	if (!ASSERT_OK(system(cmd), "iptables"))
++	if (!ASSERT_OK(system(cmd), cmd))
+ 		goto end;
+
+ 	srv_port = (mode == TEST_XDP) ? 5005 : 5006;
+diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c b/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c
+index 75550a40e029..c72083885b6d 100644
+--- a/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c
++++ b/tools/testing/selftests/bpf/prog_tests/xdp_synproxy.c
+@@ -94,12 +94,12 @@ static void test_synproxy(bool xdp)
+ 	SYS("sysctl -w net.ipv4.tcp_syncookies=2");
+ 	SYS("sysctl -w net.ipv4.tcp_timestamps=1");
+ 	SYS("sysctl -w net.netfilter.nf_conntrack_tcp_loose=0");
+-	SYS("iptables -t raw -I PREROUTING \
++	SYS("iptables-legacy -t raw -I PREROUTING \
+ 	    -i tmp1 -p tcp -m tcp --syn --dport 8080 -j CT --notrack");
+-	SYS("iptables -t filter -A INPUT \
++	SYS("iptables-legacy -t filter -A INPUT \
+ 	    -i tmp1 -p tcp -m tcp --dport 8080 -m state --state INVALID,UNTRACKED \
+ 	    -j SYNPROXY --sack-perm --timestamp --wscale 7 --mss 1460");
+-	SYS("iptables -t filter -A INPUT \
++	SYS("iptables-legacy -t filter -A INPUT \
+ 	    -i tmp1 -m state --state INVALID -j DROP");
+
+ 	ctrl_file = SYS_OUT("./xdp_synproxy --iface tmp1 --ports 8080 \
+-- 
+2.30.2
+
diff --git a/ci/vmtest/configs/DENYLIST b/ci/vmtest/configs/DENYLIST
@@ -0,0 +1,6 @@
+# TEMPORARY
+btf_dump/btf_dump: syntax
+kprobe_multi_test/bench_attach
+core_reloc/enum64val
+core_reloc/size___diff_sz
+core_reloc/type_based___diff_sz
diff --git a/ci/vmtest/configs/DENYLIST.s390x b/ci/vmtest/configs/DENYLIST.s390x
@@ -0,0 +1,5 @@
+deny_namespace                           # not yet in bpf denylist
+tc_redirect/tc_redirect_dtime            # very flaky
+lru_bug                                  # not yet in bpf-next denylist
+usdt/basic                               # failing verifier due to bounds check after LLVM update
+usdt/multispec                           # same as above
diff --git a/ci/vmtest/helpers.sh b/ci/vmtest/helpers.sh
@@ -0,0 +1,36 @@
+# $1 - start or end
+# $2 - fold identifier, no spaces
+# $3 - fold section description
+foldable() {
+  local YELLOW='\033[1;33m'
+  local NOCOLOR='\033[0m'
+  if [ $1 = "start" ]; then
+    line="::group::$2"
+    if [ ! -z "${3:-}" ]; then
+      line="$line - ${YELLOW}$3${NOCOLOR}"
+    fi
+  else
+    line="::endgroup::"
+  fi
+  echo -e "$line"
+}
+
+__print() {
+  local TITLE=""
+  if [[ -n $2 ]]; then
+      TITLE=" title=$2"
+  fi
+  echo "::$1${TITLE}::$3"
+}
+
+# $1 - title
+# $2 - message
+print_error() {
+  __print error $1 $2
+}
+
+# $1 - title
+# $2 - message
+print_notice() {
+  __print notice $1 $2
+}