[1.15] reproducible bpf unit tests, tcx downgrade, stable testdata

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "Cilium",
-  "image": "quay.io/cilium/cilium-builder:3371a65aea55999ee1de92672975c70562438f2b@sha256:4aa076f00f27302331955b9b983a2e3811bd2bba6e90eded2f0e3511c874bf80",
+  "image": "quay.io/cilium/cilium-builder:5e8053e5b2144120323f321d7963e7bebb952e43@sha256:db3600a732881404b3b491890caf9c2479d759756169efb170ab9a9958568f11",
   "workspaceFolder": "/go/src/github.com/cilium/cilium",
   "workspaceMount": "source=${localWorkspaceFolder},target=/go/src/github.com/cilium/cilium,type=bind",
   "features": {

.github/workflows/lint-bpf-checks.yaml

@@ -44,7 +44,7 @@ jobs:
             coccinelle:
               - 'contrib/coccinelle/**'
             bpf-tests-runner:
-              - 'test/bpf_tests/**'
+              - 'bpf/tests/bpftest/**'
               - 'pkg/bpf/**'
             workflow-description:
               - '.github/workflows/lint-bpf-checks.yaml'
@@ -139,32 +139,11 @@ jobs:
     name: BPF unit/integration Tests
     runs-on: ubuntu-22.04
     steps:
-      - name: Install Go
-        uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
-        with:
-          # renovate: datasource=golang-version depName=go
-          go-version: 1.21.8
-      - name: Cache LLVM and Clang
-        id: cache-llvm
-        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
-        with:
-          path: ${{ needs.set_clang_dir.outputs.clang_dir }}
-          key: llvm-10.0
-      - name: Install LLVM and Clang prerequisites
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends libtinfo5
-      - name: Install LLVM and Clang
-        uses: KyleMayes/install-llvm-action@be40c5af3a4adc3e4a03199995ab73aa37536712 # v1.9.0
-        with:
-          version: "10.0"
-          directory: ${{ needs.set_clang_dir.outputs.clang_dir }}
-          cached: ${{ steps.cache-llvm.outputs.cache-hit }}
       - name: Checkout code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
         with:
           persist-credentials: false
           fetch-depth: 0
       - name: Run BPF tests
         run: |
-          make -C test run_bpf_tests || (echo "Run 'make -C test run_bpf_tests' locally to investigate failures"; exit 1)
+          make run_bpf_tests || (echo "Run 'make run_bpf_tests' locally to investigate failures"; exit 1)

Documentation/contributing/testing/bpf.rst

@@ -27,18 +27,11 @@ To run the tests in your local environment, execute the following command from t
 
 .. code-block:: shell-session
 
-    $ make -C test run_bpf_tests
-
-The output is verbose by default. Verbose mode can be disabled by setting the ``V`` option to ``0``:
-
-.. code-block:: shell-session
-
-    $ make -C test run_bpf_tests V=0
+    $ make run_bpf_tests
 
 .. note:: 
 
-    Running BPF tests only works on Linux machines and requires admin privileges.
-    The makefile uses sudo implicitly and may prompt you for credentials.
+    Running BPF tests requires Docker and is only expected to work on Linux.
 
 Writing tests
 =============

Makefile

@@ -891,11 +891,11 @@ help: ## Display help for the Makefile, from https://www.thapaliya.com/en/writin
 .PHONY: help clean clean-container dev-doctor force generate-api generate-health-api generate-operator-api generate-hubble-api install licenses-all veryclean check-sources
 force :;
 
-# this top level run_bpf_tests target will run the bpf unit tests inside the Cilium Builder container.
-# it exists here so the entire source code repo can be mounted into the container.
-CILIUM_BUILDER_IMAGE=$(shell cat images/cilium/Dockerfile | grep "ARG CILIUM_BUILDER_IMAGE=" | cut -d"=" -f2)
-run_bpf_tests:
-	docker run -v $$(pwd):/src --privileged -w /src -e RUN_WITH_SUDO=false $(CILIUM_BUILDER_IMAGE) "make" "-C" "test/" "run_bpf_tests"
+run_bpf_tests: ## Build and run the BPF unit tests using the cilium-builder container image.
+	docker run --rm --privileged \
+		-v $$(pwd):/src -w /src \
+		$(CILIUM_BUILDER_IMAGE) \
+		"make" "-j$(shell nproc)" "-C" "bpf/tests/" "all" "run"
 
-run-builder:
+run-builder: ## Drop into a shell inside a container running the cilium-builder image.
 	docker run -it --rm -v $$(pwd):/go/src/github.com/cilium/cilium $(CILIUM_BUILDER_IMAGE) bash

Makefile.defs

@@ -65,6 +65,8 @@ endif
 
 CONSUL_IMAGE=consul:1.7.2
 
+CILIUM_BUILDER_IMAGE=$(shell cat $(ROOT_DIR)/images/cilium/Dockerfile | grep "ARG CILIUM_BUILDER_IMAGE=" | cut -d"=" -f2)
+
 export CILIUM_CLI ?= cilium
 export KUBECTL ?= kubectl
 

api/v1/Makefile

@@ -2,9 +2,6 @@
 # SPDX-License-Identifier: Apache-2.0
 include ../../Makefile.defs
 
-# Update this via images/scripts/update-cilium-builder-image.sh
-CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:3371a65aea55999ee1de92672975c70562438f2b@sha256:4aa076f00f27302331955b9b983a2e3811bd2bba6e90eded2f0e3511c874bf80
-
 .PHONY: proto
 proto:
 	$(QUIET)$(CONTAINER_ENGINE) container run --rm \

bpf/tests/Makefile

@@ -29,22 +29,22 @@ else
 LLC_FLAGS += -mcpu=v2
 endif
 
-.PHONY: all clean
+.PHONY: all clean run
 
 TEST_OBJECTS = $(patsubst %.c, %.o, $(wildcard *.c))
 
 %.o: %.ll %.i $(LIB)
-	$(ECHO_CC)
-	# Remove the .o file to force recompilation, only rely on make's caching, not clangs
+	@$(ECHO_CC)
+	@# Remove the .o file to force recompilation, only rely on make's caching, not clangs
 	rm -f $@
 	$(QUIET) ${LLC} ${LLC_FLAGS} -filetype=obj -o $@ $<
 
 %.ll: %.c
-	$(ECHO_CC)
+	@$(ECHO_CC)
 	$(QUIET) ${CLANG} ${CLANG_FLAGS} -c $< -o $@
 
 %.i: %.c
-	$(ECHO_CC)
+	@$(ECHO_CC)
 	$(QUIET) ${CLANG} ${CLANG_FLAGS} -E -c $< -o $@
 
 all: $(TEST_OBJECTS)
@@ -55,4 +55,33 @@ clean:
 	rm -f $(wildcard *.i)
 	rm -f $(wildcard *.d)
 
+BPF_TEST_FLAGS:=
+ifneq ($(shell id -u), 0)
+		BPF_TEST_FLAGS += -exec "sudo -E"
+endif
+ifeq ($(V),1)
+    BPF_TEST_FLAGS += -test.v
+endif
+ifeq ($(COVER),1)
+	ifndef COVERFORMAT
+		COVERFORMAT:=html
+	endif
+    BPF_TEST_FLAGS += -coverage-report $(ROOT_DIR)/bpf-coverage.$(COVERFORMAT) -coverage-format $(COVERFORMAT)
+ifdef NOCOVER
+    BPF_TEST_FLAGS += -no-test-coverage "$(NOCOVER)"
+endif
+endif
+ifeq ($(INSTRLOG),1)
+    BPF_TEST_FLAGS += -instrumentation-log $(ROOT_DIR)/test/bpf-instrumentation.log
+endif
+ifdef RUN
+    BPF_TEST_FLAGS += -run $(RUN)
+endif
+ifdef DUMPCTX
+    BPF_TEST_FLAGS += -dump-ctx
+endif
+
+run:
+	$(QUIET)$(GO) test ./bpftest -bpf-test-path $(ROOT_DIR)/bpf/tests $(BPF_TEST_FLAGS)
+
 -include $(TEST_OBJECTS:.o=.d)

bpf/tests/common.h

@@ -58,7 +58,7 @@
 	___bpf_apply(__bpf_log_arg, ___bpf_narg(args))(ptr, args)
 
 /* These values have to stay in sync with the enum */
-/* values in test/bpf_tests/trf.proto */
+/* values in bpf/tests/bpftest/trf.proto */
 #define TEST_ERROR 0
 #define TEST_PASS 1
 #define TEST_FAIL 2

images/builder/update-cilium-builder-image.sh

@@ -16,7 +16,7 @@ cd "${root_dir}"
 image="quay.io/cilium/cilium-builder"
 
 # shellcheck disable=SC2207
-used_by=($(git grep -l CILIUM_BUILDER_IMAGE= images/*/Dockerfile) "test/k8s/manifests/demo-customcalls.yaml" "api/v1/Makefile")
+used_by=($(git grep -l CILIUM_BUILDER_IMAGE= images/*/Dockerfile) "test/k8s/manifests/demo-customcalls.yaml")
 
 for i in "${used_by[@]}" ; do
   sed -E "s#(CILIUM_BUILDER_IMAGE=|image: )${image}:.*\$#\1${image_full}#" "${i}" > "${i}.sedtmp" && mv "${i}.sedtmp" "${i}"

images/cilium/Dockerfile

@@ -1,7 +1,7 @@
 # Copyright Authors of Cilium
 # SPDX-License-Identifier: Apache-2.0
 
-ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:3371a65aea55999ee1de92672975c70562438f2b@sha256:4aa076f00f27302331955b9b983a2e3811bd2bba6e90eded2f0e3511c874bf80
+ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:5e8053e5b2144120323f321d7963e7bebb952e43@sha256:db3600a732881404b3b491890caf9c2479d759756169efb170ab9a9958568f11
 ARG CILIUM_RUNTIME_IMAGE=quay.io/cilium/cilium-runtime:6fce5345b0b7bcfa40bbf7c050122889c21868fa@sha256:8cf13516bf371182bd1a26e945a050fd73420d433e8644ecd3e09f3dac86977c
 
 # cilium-envoy from github.com/cilium/proxy

images/hubble-relay/Dockerfile

@@ -12,7 +12,7 @@
 # https://raw.githubusercontent.com/GoogleContainerTools/distroless/main/cosign.pub
 ARG BASE_IMAGE=gcr.io/distroless/static-debian11:nonroot@sha256:6732c3975d97fac664a5ed15a81a5915e023a7b5a7b58195e733c60b8dc7e684
 ARG GOLANG_IMAGE=docker.io/library/golang:1.21.8@sha256:856073656d1a517517792e6cdd2f7a5ef080d3ca2dff33e518c8412f140fdd2d
-ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:3371a65aea55999ee1de92672975c70562438f2b@sha256:4aa076f00f27302331955b9b983a2e3811bd2bba6e90eded2f0e3511c874bf80
+ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:5e8053e5b2144120323f321d7963e7bebb952e43@sha256:db3600a732881404b3b491890caf9c2479d759756169efb170ab9a9958568f11
 
 # BUILDPLATFORM is an automatic platform ARG enabled by Docker BuildKit.
 # Represents the plataform where the build is happening, do not mix with

images/operator/Dockerfile

@@ -4,7 +4,7 @@
 ARG BASE_IMAGE=scratch
 ARG GOLANG_IMAGE=docker.io/library/golang:1.21.8@sha256:856073656d1a517517792e6cdd2f7a5ef080d3ca2dff33e518c8412f140fdd2d
 ARG ALPINE_IMAGE=docker.io/library/alpine:3.19.1@sha256:c5b1261d6d3e43071626931fc004f70149baeba2c8ec672bd4f27761f8e1ad6b
-ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:3371a65aea55999ee1de92672975c70562438f2b@sha256:4aa076f00f27302331955b9b983a2e3811bd2bba6e90eded2f0e3511c874bf80
+ARG CILIUM_BUILDER_IMAGE=quay.io/cilium/cilium-builder:5e8053e5b2144120323f321d7963e7bebb952e43@sha256:db3600a732881404b3b491890caf9c2479d759756169efb170ab9a9958568f11
 
 # BUILDPLATFORM is an automatic platform ARG enabled by Docker BuildKit.
 # Represents the plataform where the build is happening, do not mix with

pkg/bpf/bpffs_linux.go

@@ -6,6 +6,7 @@
 package bpf
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -64,6 +65,15 @@ func MkdirBPF(path string) error {
 	return os.MkdirAll(path, 0755)
 }
 
+// Remove path ignoring ErrNotExist.
+func Remove(path string) error {
+	err := os.RemoveAll(path)
+	if err != nil && !errors.Is(err, os.ErrNotExist) {
+		return fmt.Errorf("removing bpffs directory at %s: %w", path, err)
+	}
+	return err
+}
+
 func tcPathFromMountInfo(name string) string {
 	readMountInfo.Do(func() {
 		mountInfos, err := mountinfo.GetMountInfo()

pkg/bpf/collection.go

@@ -59,6 +59,7 @@ func removeUnreachableTailcalls(spec *ebpf.CollectionSpec) error {
 	tailcalls := make(map[uint32]*TailCall)
 
 	const (
+		// Corresponds to CILIUM_MAP_CALLS.
 		cilium_calls_map = 2
 	)
 

pkg/bpf/testdata/Makefile

@@ -1,9 +1,27 @@
 include ../../../Makefile.defs
 
-.PHONY: build
+CLANG := clang
+STRIP := llvm-objcopy
+CFLAGS := -O2 -g -Wall -Werror -I -I/usr/include -I$(ROOT_DIR)/bpf -I$(ROOT_DIR)/bpf/include $(CFLAGS)
 
-CILIUM_BUILDER_IMAGE=$(shell cat $(ROOT_DIR)/images/cilium/Dockerfile | grep "ARG CILIUM_BUILDER_IMAGE=" | cut -d"=" -f2)
+TESTDATA_DIR := $(ROOT_DIR)/pkg/bpf/testdata
+UIDGID := $(shell stat -c '%u:%g' ${ROOT_DIR})
 
-##@ Default
-build:
-	docker run --rm -v $(ROOT_DIR):/cilium $(CILIUM_BUILDER_IMAGE) clang -target bpf -Wall -O2 -g -I -I/usr/include -I/cilium/bpf -I/cilium/bpf/include -c /cilium/pkg/bpf/testdata/unreachable-tailcall.c -o /cilium/pkg/bpf/testdata/unreachable-tailcall.o
+TARGETS := \
+	unreachable-tailcall
+
+.PHONY: docker build clean
+
+.DEFAULT_GOAL: docker
+docker: ## Build the testdata binaries using the cilium-builder Docker image.
+	docker run --rm -u "$(UIDGID)" -v "$(ROOT_DIR):/cilium" $(CILIUM_BUILDER_IMAGE) make -C /cilium/pkg/bpf/testdata clean build
+
+clean: ## Remove testdata artifacts.
+	find "$(TESTDATA_DIR)" -name "*.o" -delete
+
+## Build the testdata binaries using the host's toolchain.
+build: $(addsuffix .o,$(TARGETS))
+
+%.o: %.c
+	$(CLANG) $(CFLAGS) -target bpfel -c $< -o $@
+	$(STRIP) -g $@

pkg/bpf/testdata/unreachable-tailcall.c

@@ -1,59 +1,63 @@
+#include <bpf/loader.h>
+#include <bpf/section.h>
+
 #include <bpf/ctx/skb.h>
-#include <ep_config.h>
-#include <node_config.h>
+#include <bpf/tailcall.h>
+
+#define CILIUM_MAP_CALLS 2
+
+volatile const int global_var = 0;
 
-#define __NR_CPUS__ 1
-#include "lib/maps.h"
+struct bpf_elf_map __section_maps cilium_calls_test = {
+	.type		= BPF_MAP_TYPE_PROG_ARRAY,
+	.id		= CILIUM_MAP_CALLS,
+	.size_key	= sizeof(__u32),
+	.size_value	= sizeof(__u32),
+	.max_elem	= 5,
+};
 
-#define TAIL_A 1
-#define TAIL_B 2
-#define TAIL_C 3
-#define TAIL_D 4
-#define TAIL_E 5
+#define TAIL_A 0
+#define TAIL_B 1
+#define TAIL_C 2
+#define TAIL_D 3
+#define TAIL_E 4
 
 __section_tail(CILIUM_MAP_CALLS, TAIL_E)
-static int e(struct __sk_buff *ctx) {
+static int e(void *ctx) {
         return 0;
 }
 
 __section_tail(CILIUM_MAP_CALLS, TAIL_D)
-static int d(struct __sk_buff *ctx) {
-        tail_call_static(ctx, CALLS_MAP, TAIL_E);
+static int d(void *ctx) {
+        tail_call_static(ctx, cilium_calls_test, TAIL_E);
         return 0;
 }
 
 __section_tail(CILIUM_MAP_CALLS, TAIL_C)
-static int c(struct __sk_buff *ctx) {
+static int c(void *ctx) {
         return 0;
 }
 
 __section_tail(CILIUM_MAP_CALLS, TAIL_B)
-static int b(struct __sk_buff *ctx) {
-        tail_call_static(ctx, CALLS_MAP, TAIL_C);
+static int b(void *ctx) {
+        tail_call_static(ctx, cilium_calls_test, TAIL_C);
         return 0;
 }
 
-volatile const int just_a_var = 0;
 
 __section_tail(CILIUM_MAP_CALLS, TAIL_A)
-static int a(struct __sk_buff *ctx) {
-        void *data = (void*)(long long)ctx->data;
-        void *data_end = (void*)(long long)ctx->data_end;
-
-        if (data + 1 > data_end)
-                return 0;
-
-        if (just_a_var == 0x01) {
-                tail_call_static(ctx, CALLS_MAP, TAIL_B);
+static int a(void *ctx) {
+        if (global_var == 0x01) {
+                tail_call_static(ctx, cilium_calls_test, TAIL_B);
         } else {
-                tail_call_static(ctx, CALLS_MAP, TAIL_C);
+                tail_call_static(ctx, cilium_calls_test, TAIL_C);
         }
 
         return 0;
 }
 
 __section("tc")
-static int cil_entry(struct __sk_buff *ctx) {
-        tail_call_static(ctx, CALLS_MAP, TAIL_A);
+static int cil_entry(void *ctx) {
+        tail_call_static(ctx, cilium_calls_test, TAIL_A);
         return 0;
 }