[BPF] Handle unreachable with a kfunc call

Yonghong Song · Yonghong Song · commit c78aba3363c2 · 2025-05-22T10:29:29.000-07:00
Currently, middle-end generates 'unreachable' insn if the compiler feels the code is indeed unreachable or the code becomes invalid due to some optimizaiton (e.g. code optimization with uninitialized variables). Right now BPF backend ignores 'unreachable' insn during selectiondag lowering. For cases where 'unreachable' is due to invalid code transformation, such a signal will be missed. Later on, users needs some effort to debug it which impacts developer productivity. This patch enabled selectiondag lowering for 'unreachable' insn. Previous attempt ([1]) tries to have a backend IR pass to filter out 'unreachable' insns in a number of cases. But such pattern matching may misalign with future middle-end optimization with 'unreachable' insns. This patch takes a different approach. The 'unreachable' insn is lowered with special encoding in bpf object file and verifier will do proper verification for the bpf prog. More specifically, the 'unreachable' insn is replaced by a '__bpf_unreachable' function. This function will be a kfunc (in ".ksyms" section) with a weak attribute, but does not have definition. The function is also present in prog btf. This way, the extern '__bpf_unreachable' can be handled properly in libbpf func poison_kfunc_call(). The name '__bpf_unreachable' is chosen to satisfy reserved identifier requirement. I tested this patch on bpf selftests and all tests are passed. I also tried original example in [2] and the error is properly detected by verifier: func#0 @0 last insn is not an exit or jmp In another internal sched-ext bpf prog, with the patch we have bpf code: Disassembly of section .text: 0000000000000000 <scx_storage_init_single>: ; { 0: bc 13 00 00 00 00 00 00 w3 = w1 1: b4 01 00 00 00 00 00 00 w1 = 0x0 ; const u32 zero = 0; ... 0000000000003a80 <create_dom>: ; { 1872: bc 16 00 00 00 00 00 00 w6 = w1 ; bpf_printk("dom_id %d", dom_id); 1873: 18 01 00 00 3f 00 00 00 00 00 00 00 00 00 00 00 r1 = 0x3f ll 0000000000003a88: R_BPF_64_64 .rodata 1875: b4 02 00 00 0a 00 00 00 w2 = 0xa 1876: bc 63 00 00 00 00 00 00 w3 = w6 1877: 85 00 00 00 06 00 00 00 call 0x6 ; ret = scx_bpf_create_dsq(dom_id, 0); 1878: bc 61 00 00 00 00 00 00 w1 = w6 1879: b4 02 00 00 00 00 00 00 w2 = 0x0 1880: 85 10 00 00 ff ff ff ff call -0x1 0000000000003ac0: R_BPF_64_32 scx_bpf_create_dsq ; domc->node_cpumask = node_data[node_id]; 1881: 85 10 00 00 ff ff ff ff call -0x1 0000000000003ac8: R_BPF_64_32 __bpf_unreachable <END> The verifier can easily report the error too. A bpf flag `-bpf-disable-trap-unreachable` is introduced to disable trapping unreachable. [1] #126858 [2] https://github.com/msune/clang_bpf/blob/main/Makefile#L3
diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h
@@ -22,6 +22,8 @@ class BPFTargetMachine;
 class InstructionSelector;
 class PassRegistry;
 
+static const char *BPF_UNREACHABLE = "__bpf_unreachable";
+
 ModulePass *createBPFCheckAndAdjustIR();
 
 FunctionPass *createBPFISelDag(BPFTargetMachine &TM);
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -21,8 +21,10 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
 #include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/IR/DIBuilder.h"
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/DiagnosticPrinter.h"
+#include "llvm/IR/Module.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/MathExtras.h"
@@ -68,6 +70,8 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
   setOperationAction(ISD::BRIND, MVT::Other, Expand);
   setOperationAction(ISD::BRCOND, MVT::Other, Expand);
 
+  setOperationAction(ISD::TRAP, MVT::Other, Custom);
+
   setOperationAction({ISD::GlobalAddress, ISD::ConstantPool}, MVT::i64, Custom);
 
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
@@ -326,6 +330,8 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::ATOMIC_LOAD:
   case ISD::ATOMIC_STORE:
     return LowerATOMIC_LOAD_STORE(Op, DAG);
+  case ISD::TRAP:
+    return LowerTRAP(Op, DAG);
   }
 }
 
@@ -521,10 +527,12 @@ SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
                                         G->getOffset(), 0);
   } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
-    Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
-    fail(CLI.DL, DAG,
-         Twine("A call to built-in function '" + StringRef(E->getSymbol()) +
-               "' is not supported."));
+    if (StringRef(E->getSymbol()) != BPF_UNREACHABLE) {
+      Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
+      fail(CLI.DL, DAG,
+           Twine("A call to built-in function '" + StringRef(E->getSymbol()) +
+                 "' is not supported."));
+    }
   }
 
   // Returns a chain & a flag for retval copy to use.
@@ -726,6 +734,52 @@ SDValue BPFTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
   return Op;
 }
 
+static Function *createBPFUnreachable(Module *M) {
+  if (auto *Fn = M->getFunction(BPF_UNREACHABLE))
+    return Fn;
+
+  FunctionType *FT = FunctionType::get(Type::getVoidTy(M->getContext()), false);
+  Function *NewF = Function::Create(FT, GlobalValue::ExternalWeakLinkage,
+                                    BPF_UNREACHABLE, M);
+  NewF->setDSOLocal(true);
+  NewF->setCallingConv(CallingConv::C);
+  NewF->setSection(".ksyms");
+
+  if (M->debug_compile_units().empty())
+    return NewF;
+
+  DIBuilder DBuilder(*M);
+  DITypeRefArray ParamTypes =
+      DBuilder.getOrCreateTypeArray({nullptr /*void return*/});
+  DISubroutineType *FuncType = DBuilder.createSubroutineType(ParamTypes);
+  DICompileUnit *CU = *M->debug_compile_units_begin();
+  DISubprogram *SP = DBuilder.createFunction(
+      CU, BPF_UNREACHABLE, BPF_UNREACHABLE, nullptr, 0, FuncType, 0,
+      DINode::FlagZero, DISubprogram::SPFlagZero);
+  NewF->setSubprogram(SP);
+  return NewF;
+}
+
+SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+  TargetLowering::CallLoweringInfo CLI(DAG);
+  SmallVector<SDValue> InVals;
+  SDNode *N = Op.getNode();
+  SDLoc DL(N);
+
+  Function *Fn = createBPFUnreachable(MF.getFunction().getParent());
+  auto PtrVT = getPointerTy(MF.getDataLayout());
+  CLI.Callee = DAG.getTargetGlobalAddress(Fn, DL, PtrVT);
+  CLI.Chain = N->getOperand(0);
+  CLI.IsTailCall = false;
+  CLI.CallConv = CallingConv::C;
+  CLI.IsVarArg = false;
+  CLI.DL = DL;
+  CLI.NoMerge = false;
+  CLI.DoesNotReturn = true;
+  return LowerCall(CLI, InVals);
+}
+
 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
   switch ((BPFISD::NodeType)Opcode) {
   case BPFISD::FIRST_NUMBER:
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -80,6 +80,7 @@ class BPFTargetLowering : public TargetLowering {
   SDValue LowerATOMIC_LOAD_STORE(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerTRAP(SDValue Op, SelectionDAG &DAG) const;
 
   template <class NodeTy>
   SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
diff --git a/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
@@ -320,6 +320,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
   bool adjustBranch();
   bool insertMissingCallerSavedSpills();
   bool removeMayGotoZero();
+  bool addExitAfterUnreachable();
 
 public:
 
@@ -336,6 +337,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
       Changed = adjustBranch() || Changed;
     Changed |= insertMissingCallerSavedSpills();
     Changed |= removeMayGotoZero();
+    Changed |= addExitAfterUnreachable();
     return Changed;
   }
 };
@@ -734,6 +736,20 @@ bool BPFMIPreEmitPeephole::removeMayGotoZero() {
   return Changed;
 }
 
+// If the last insn in a funciton is 'JAL &bpf_unreachable', let us add an
+// 'exit' insn after that insn. This will ensure no fallthrough at the last
+// insn, making kernel verification easier.
+bool BPFMIPreEmitPeephole::addExitAfterUnreachable() {
+  MachineBasicBlock &MBB = MF->back();
+  MachineInstr &MI = MBB.back();
+  if (MI.getOpcode() != BPF::JAL || !MI.getOperand(0).isGlobal() ||
+      MI.getOperand(0).getGlobal()->getName() != BPF_UNREACHABLE)
+    return false;
+
+  BuildMI(&MBB, MI.getDebugLoc(), TII->get(BPF::RET));
+  return true;
+}
+
 } // end default namespace
 
 INITIALIZE_PASS(BPFMIPreEmitPeephole, "bpf-mi-pemit-peephole",
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -37,6 +37,10 @@ static cl::
 opt<bool> DisableMIPeephole("disable-bpf-peephole", cl::Hidden,
                             cl::desc("Disable machine peepholes for BPF"));
 
+static cl::opt<bool>
+    DisableCheckUnreachable("bpf-disable-trap-unreachable", cl::Hidden,
+                            cl::desc("Disable Trap Unreachable for BPF"));
+
 extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() {
   // Register the target.
   RegisterTargetMachine<BPFTargetMachine> X(getTheBPFleTarget());
@@ -77,6 +81,11 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT,
                                getEffectiveCodeModel(CM, CodeModel::Small), OL),
       TLOF(std::make_unique<TargetLoweringObjectFileELF>()),
       Subtarget(TT, std::string(CPU), std::string(FS), *this) {
+  if (!DisableCheckUnreachable) {
+    this->Options.TrapUnreachable = true;
+    this->Options.NoTrapAfterNoreturn = true;
+  }
+
   initAsmInfo();
 
   BPFMCAsmInfo *MAI =
diff --git a/llvm/test/CodeGen/BPF/BTF/bpf_unreachable.ll b/llvm/test/CodeGen/BPF/BTF/bpf_unreachable.ll
@@ -0,0 +1,53 @@
+; RUN: llc -mtriple=bpfel -mcpu=v3 -filetype=obj -o %t1 %s
+; RUN: llvm-objcopy --dump-section='.BTF'=%t2 %t1
+; RUN: %python %p/print_btf.py %t2 | FileCheck -check-prefixes=CHECK-BTF %s
+; RUN: llc -mtriple=bpfel -mcpu=v3 < %s | FileCheck -check-prefixes=CHECK %s
+
+define void @foo() {
+entry:
+  tail call void @bar()
+  unreachable
+}
+
+; CHECK:      foo:
+; CHECK-NEXT: .Lfunc_begin0:
+; CHECK-NEXT:   .cfi_startproc
+; CHECK-NEXT: # %bb.0:
+; CHECK-NEXT:    call bar
+; CHECK-NEXT:    call __bpf_unreachable
+; CHECK-NEXT:    exit
+; CHECK-NEXT: .Lfunc_end0:
+
+define void @buz() #0 {
+entry:
+  tail call void asm sideeffect "r0 = r1; exit;", ""()
+  unreachable
+}
+
+; CHECK: buz:
+; CHECK-NEXT: .Lfunc_begin1:
+; CHECK-NEXT:   .cfi_startproc
+; CHECK-NEXT: # %bb.0:
+; CHECK-NEXT:   #APP
+; CHECK-NEXT:   r0 = r1
+; CHECK-NEXT:   exit
+; CHECK-EMPTY:
+; CHECK-NEXT:   #NO_APP
+; CHECK-NEXT: .Lfunc_end1:
+
+; CHECK-BTF: [1] FUNC_PROTO '(anon)' ret_type_id=0 vlen=0
+; CHECK-BTF: [2] FUNC '__bpf_unreachable' type_id=1 linkage=extern
+; CHECK-BTF: [3] DATASEC '.ksyms' size=0 vlen=1
+; CHECK-BTF:  type_id=2 offset=0 size=0
+
+declare dso_local void @bar()
+
+attributes #0 = { naked }
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!2, !3}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, emissionKind: FullDebug)
+!1 = !DIFile(filename: "test.c", directory: "/some/dir")
+!2 = !{i32 7, !"Dwarf Version", i32 5}
+!3 = !{i32 2, !"Debug Info Version", i32 3}
