Add support for weak kfuncs

elf_reader.go

@@ -26,7 +26,12 @@ type kconfigMeta struct {
 	Offset uint32
 }
 
-type kfuncMeta struct{}
+type kfuncMetaKey struct{}
+
+type kfuncMeta struct {
+	Binding elf.SymBind
+	Func    *btf.Func
+}
 
 // elfCode is a convenience to reduce the amount of arguments that have to
 // be passed around explicitly. You should treat its contents as immutable.
@@ -600,7 +605,7 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err
 	// function declarations, as well as extern kfunc declarations using __ksym
 	// and extern kconfig variables declared using __kconfig.
 	case undefSection:
-		if bind != elf.STB_GLOBAL {
+		if bind != elf.STB_GLOBAL && bind != elf.STB_WEAK {
 			return fmt.Errorf("asm relocation: %s: %w: %s", name, errUnsupportedBinding, bind)
 		}
 
@@ -610,13 +615,25 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err
 
 		kf := ec.kfuncs[name]
 		switch {
-		// If a Call instruction is found and the datasec has a btf.Func with a Name
-		// that matches the symbol name we mark the instruction as a call to a kfunc.
+		// If a Call / DWordLoad instruction is found and the datasec has a btf.Func with a Name
+		// that matches the symbol name we mark the instruction as a referencing a kfunc.
 		case kf != nil && ins.OpCode.JumpOp() == asm.Call:
-			ins.Metadata.Set(kfuncMeta{}, kf)
+			ins.Metadata.Set(kfuncMetaKey{}, &kfuncMeta{
+				Func:    kf,
+				Binding: bind,
+			})
+
 			ins.Src = asm.PseudoKfuncCall
 			ins.Constant = -1
 
+		case kf != nil && ins.OpCode.IsDWordLoad():
+			ins.Metadata.Set(kfuncMetaKey{}, &kfuncMeta{
+				Func:    kf,
+				Binding: bind,
+			})
+
+			ins.Constant = 0
+
 		// If no kconfig map is found, this must be a symbol reference from inline
 		// asm (see testdata/loader.c:asm_relocation()) or a call to a forward
 		// function declaration (see testdata/fwd_decl.c). Don't interfere, These
@@ -626,6 +643,10 @@ func (ec *elfCode) relocateInstruction(ins *asm.Instruction, rel elf.Symbol) err
 		// require it to contain the symbol to disambiguate between inline asm
 		// relos and kconfigs.
 		case ec.kconfig != nil && ins.OpCode.IsDWordLoad():
+			if bind != elf.STB_GLOBAL {
+				return fmt.Errorf("asm relocation: %s: %w: %s", name, errUnsupportedBinding, bind)
+			}
+
 			for _, vsi := range ec.kconfig.Value.(*btf.Datasec).Vars {
 				if vsi.Type.(*btf.Var).Name != rel.Name {
 					continue

elf_reader_test.go

@@ -750,6 +750,33 @@ func TestKfunc(t *testing.T) {
 	})
 }
 
+func TestWeakKfunc(t *testing.T) {
+	testutils.SkipOnOldKernel(t, "5.18", "kfunc support")
+	testutils.Files(t, testutils.Glob(t, "testdata/kfunc-e*.elf"), func(t *testing.T, file string) {
+		spec, err := LoadCollectionSpec(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if spec.ByteOrder != internal.NativeEndian {
+			return
+		}
+
+		var obj struct {
+			Missing *Program `ebpf:"weak_kfunc_missing"`
+			Calling *Program `ebpf:"call_weak_kfunc"`
+		}
+
+		err = spec.LoadAndAssign(&obj, nil)
+		testutils.SkipIfNotSupported(t, err)
+		if err != nil {
+			t.Fatalf("%+v", err)
+		}
+		defer obj.Missing.Close()
+		defer obj.Calling.Close()
+	})
+}
+
 func TestInvalidKfunc(t *testing.T) {
 	testutils.SkipOnOldKernel(t, "5.18", "kfunc support")
 

linker.go

@@ -1,6 +1,7 @@
 package ebpf
 
 import (
+	"debug/elf"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -259,6 +260,10 @@ func fixupAndValidate(insns asm.Instructions) error {
 	return nil
 }
 
+// POISON_CALL_KFUNC_BASE in libbpf.
+// https://github.com/libbpf/libbpf/blob/2778cbce609aa1e2747a69349f7f46a2f94f0522/src/libbpf.c#L5767
+const kfuncCallPoisonBase = 2002000000
+
 // fixupKfuncs loops over all instructions in search for kfunc calls.
 // If at least one is found, the current kernels BTF and module BTFis are searched to set Instruction.Constant
 // and Instruction.Offset to the correct values.
@@ -272,7 +277,7 @@ func fixupKfuncs(insns asm.Instructions) (_ handles, err error) {
 	iter := insns.Iterate()
 	for iter.Next() {
 		ins := iter.Ins
-		if ins.IsKfuncCall() {
+		if metadata := ins.Metadata.Get(kfuncMetaKey{}); metadata != nil {
 			goto fixups
 		}
 	}
@@ -292,7 +297,8 @@ fixups:
 	for {
 		ins := iter.Ins
 
-		if !ins.IsKfuncCall() {
+		metadata := ins.Metadata.Get(kfuncMetaKey{})
+		if metadata == nil {
 			if !iter.Next() {
 				// break loop if this was the last instruction in the stream.
 				break
@@ -301,15 +307,34 @@ fixups:
 		}
 
 		// check meta, if no meta return err
-		kfm, _ := ins.Metadata.Get(kfuncMeta{}).(*btf.Func)
+		kfm, _ := metadata.(*kfuncMeta)
 		if kfm == nil {
-			return nil, fmt.Errorf("kfunc call has no kfuncMeta")
+			return nil, fmt.Errorf("kfuncMetaKey doesn't contain kfuncMeta")
 		}
 
 		target := btf.Type((*btf.Func)(nil))
-		spec, module, err := findTargetInKernel(kernelSpec, kfm.Name, &target)
+		spec, module, err := findTargetInKernel(kernelSpec, kfm.Func.Name, &target)
+		if kfm.Binding == elf.STB_WEAK && errors.Is(err, btf.ErrNotFound) {
+			if ins.IsKfuncCall() {
+				// If the kfunc call is weak and not found, poison the call. Use a recognizable constant
+				// to make it easier to debug. And set src to zero so the verifier doesn't complain
+				// about the invalid imm/offset values before dead-code elimination.
+				ins.Constant = kfuncCallPoisonBase
+				ins.Src = 0
+			} else if ins.OpCode.IsDWordLoad() {
+				// If the kfunc DWordLoad is weak and not found, set its address to 0.
+				ins.Constant = 0
+				ins.Src = 0
+			} else {
+				return nil, fmt.Errorf("only kfunc calls and dword loads may have kfunc metadata")
+			}
+
+			iter.Next()
+			continue
+		}
+		// Error on non-weak kfunc not found.
 		if errors.Is(err, btf.ErrNotFound) {
-			return nil, fmt.Errorf("kfunc %q: %w", kfm.Name, ErrNotSupported)
+			return nil, fmt.Errorf("kfunc %q: %w", kfm.Func.Name, ErrNotSupported)
 		}
 		if err != nil {
 			return nil, err
@@ -320,8 +345,8 @@ fixups:
 			return nil, err
 		}
 
-		if err := btf.CheckTypeCompatibility(kfm.Type, target.(*btf.Func).Type); err != nil {
-			return nil, &incompatibleKfuncError{kfm.Name, err}
+		if err := btf.CheckTypeCompatibility(kfm.Func.Type, target.(*btf.Func).Type); err != nil {
+			return nil, &incompatibleKfuncError{kfm.Func.Name, err}
 		}
 
 		id, err := spec.TypeID(target)

testdata/common.h

@@ -20,6 +20,15 @@ enum libbpf_tristate {
 
 #define __kconfig __attribute__((section(".kconfig")))
 #define __ksym __attribute__((section(".ksyms")))
+#ifndef __weak
+#define __weak __attribute__((weak))
+#endif
+
+#define bpf_ksym_exists(sym) \
+	({ \
+		_Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \
+		!!sym; \
+	})
 
 #define BPF_MAP_TYPE_HASH (1)
 #define BPF_MAP_TYPE_ARRAY (2)

testdata/kfunc.c

@@ -8,6 +8,7 @@ struct __sk_buff {};
 struct nf_conn {};
 struct bpf_sock_tuple {};
 struct bpf_ct_opts {};
+struct bpf_cpumask {};
 
 extern struct nf_conn *bpf_skb_ct_lookup(struct __sk_buff *, struct bpf_sock_tuple *, uint32_t, struct bpf_ct_opts *, uint32_t) __ksym;
 extern void bpf_ct_release(struct nf_conn *) __ksym;
@@ -29,3 +30,29 @@ __section("fentry/bpf_fentry_test2") int benchmark() {
 	// the kernel when benchmarking the loader.
 	return bpf_fentry_test1(0);
 }
+
+extern void invalid_kfunc(void) __ksym __weak;
+
+extern struct bpf_cpumask *bpf_cpumask_create(void) __ksym __weak;
+extern void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym __weak;
+
+__section("tp_btf/task_newtask") int weak_kfunc_missing(void *ctx) {
+	if (bpf_ksym_exists(invalid_kfunc)) {
+		invalid_kfunc();
+		return 0;
+	}
+
+	return 1;
+}
+
+__section("tp_btf/task_newtask") int call_weak_kfunc(void *ctx) {
+	if (bpf_ksym_exists(bpf_cpumask_create)) {
+		struct bpf_cpumask *mask = bpf_cpumask_create();
+		if (mask)
+			bpf_cpumask_release(mask);
+
+		return 1;
+	}
+
+	return 0;
+}