diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h index 966a3b0cf26e66..82ac091fa7fa36 100644 --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -16,6 +16,7 @@ namespace llvm { class BPFTargetMachine; +ModulePass *createBPFAdjustOpt(); ModulePass *createBPFCheckAndAdjustIR(); FunctionPass *createBPFAbstractMemberAccess(BPFTargetMachine *TM); @@ -27,6 +28,7 @@ FunctionPass *createBPFMIPeepholeTruncElimPass(); FunctionPass *createBPFMIPreEmitPeepholePass(); FunctionPass *createBPFMIPreEmitCheckingPass(); +void initializeBPFAdjustOptPass(PassRegistry&); void initializeBPFCheckAndAdjustIRPass(PassRegistry&); void initializeBPFAbstractMemberAccessLegacyPassPass(PassRegistry &); diff --git a/llvm/lib/Target/BPF/BPFAdjustOpt.cpp b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp new file mode 100644 index 00000000000000..8efaa9d72b5762 --- /dev/null +++ b/llvm/lib/Target/BPF/BPFAdjustOpt.cpp @@ -0,0 +1,310 @@ +//===---------------- BPFAdjustOpt.cpp - Adjust Optimization --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Adjust optimization to make the code more kernel verifier friendly. +// +//===----------------------------------------------------------------------===// + +#include "BPF.h" +#include "BPFCORE.h" +#include "BPFTargetMachine.h" +#include "llvm/IR/Instruction.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/User.h" +#include "llvm/IR/Value.h" +#include "llvm/Pass.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" + +#define DEBUG_TYPE "bpf-adjust-opt" + +using namespace llvm; + +static cl::opt + DisableBPFserializeICMP("bpf-disable-serialize-icmp", cl::Hidden, + cl::desc("BPF: Disable Serializing ICMP insns."), + cl::init(false)); + +static cl::opt DisableBPFavoidSpeculation( + "bpf-disable-avoid-speculation", cl::Hidden, + cl::desc("BPF: Disable Avoiding Speculative Code Motion."), + cl::init(false)); + +namespace { + +class BPFAdjustOpt final : public ModulePass { + struct PassThroughInfo { + Instruction *Input; + Instruction *UsedInst; + uint32_t OpIdx; + PassThroughInfo(Instruction *I, Instruction *U, uint32_t Idx) + : Input(I), UsedInst(U), OpIdx(Idx) {} + }; + +public: + static char ID; + Module *Mod; + + BPFAdjustOpt() : ModulePass(ID) {} + bool runOnModule(Module &M) override; + +private: + SmallVector PassThroughs; + + void adjustBasicBlock(BasicBlock &BB); + bool serializeICMPCrossBB(BasicBlock &BB); + void adjustInst(Instruction &I); + bool serializeICMPInBB(Instruction &I); + bool avoidSpeculation(Instruction &I); + bool insertPassThrough(); +}; + +} // End anonymous namespace + +char BPFAdjustOpt::ID = 0; +INITIALIZE_PASS(BPFAdjustOpt, "bpf-adjust-opt", "BPF Adjust Optimization", + false, false) + +ModulePass *llvm::createBPFAdjustOpt() { return new BPFAdjustOpt(); } + +bool BPFAdjustOpt::runOnModule(Module &M) { + Mod = &M; + for (Function &F : M) + for (auto &BB : F) { + adjustBasicBlock(BB); + for (auto &I : BB) + adjustInst(I); + } + + return insertPassThrough(); +} + +bool BPFAdjustOpt::insertPassThrough() { + for (auto &Info : PassThroughs) { + auto *CI = BPFCoreSharedInfo::insertPassThrough( + Mod, Info.UsedInst->getParent(), Info.Input, Info.UsedInst); + Info.UsedInst->setOperand(Info.OpIdx, CI); + } + + return !PassThroughs.empty(); +} + +// To avoid combining conditionals in the same basic block by +// instrcombine optimization. +bool BPFAdjustOpt::serializeICMPInBB(Instruction &I) { + // For: + // comp1 = icmp ...; + // comp2 = icmp ...; + // ... or comp1 comp2 ... + // changed to: + // comp1 = icmp ...; + // comp2 = icmp ...; + // new_comp1 = __builtin_bpf_passthrough(seq_num, comp1) + // ... or new_comp1 comp2 ... + if (I.getOpcode() != Instruction::Or) + return false; + auto *Icmp1 = dyn_cast(I.getOperand(0)); + if (!Icmp1) + return false; + auto *Icmp2 = dyn_cast(I.getOperand(1)); + if (!Icmp2) + return false; + + Value *Icmp1Op0 = Icmp1->getOperand(0); + Value *Icmp2Op0 = Icmp2->getOperand(0); + if (Icmp1Op0 != Icmp2Op0) + return false; + + // Now we got two icmp instructions which feed into + // an "or" instruction. + PassThroughInfo Info(Icmp1, &I, 0); + PassThroughs.push_back(Info); + return true; +} + +// To avoid combining conditionals in the same basic block by +// instrcombine optimization. +bool BPFAdjustOpt::serializeICMPCrossBB(BasicBlock &BB) { + // For: + // B1: + // comp1 = icmp ...; + // if (comp1) goto B2 else B3; + // B2: + // comp2 = icmp ...; + // if (comp2) goto B4 else B5; + // B4: + // ... + // changed to: + // B1: + // comp1 = icmp ...; + // comp1 = __builtin_bpf_passthrough(seq_num, comp1); + // if (comp1) goto B2 else B3; + // B2: + // comp2 = icmp ...; + // if (comp2) goto B4 else B5; + // B4: + // ... + + // Check basic predecessors, if two of them (say B1, B2) are using + // icmp instructions to generate conditions and one is the predesessor + // of another (e.g., B1 is the predecessor of B2). Add a passthrough + // barrier after icmp inst of block B1. + BasicBlock *B2 = BB.getSinglePredecessor(); + if (!B2) + return false; + + BasicBlock *B1 = B2->getSinglePredecessor(); + if (!B1) + return false; + + Instruction *TI = B2->getTerminator(); + auto *BI = dyn_cast(TI); + if (!BI || !BI->isConditional()) + return false; + auto *Cond = dyn_cast(BI->getCondition()); + if (!Cond || B2->getFirstNonPHI() != Cond) + return false; + Value *B2Op0 = Cond->getOperand(0); + auto Cond2Op = Cond->getPredicate(); + + TI = B1->getTerminator(); + BI = dyn_cast(TI); + if (!BI || !BI->isConditional()) + return false; + Cond = dyn_cast(BI->getCondition()); + if (!Cond) + return false; + Value *B1Op0 = Cond->getOperand(0); + auto Cond1Op = Cond->getPredicate(); + + if (B1Op0 != B2Op0) + return false; + + if (Cond1Op == ICmpInst::ICMP_SGT || Cond1Op == ICmpInst::ICMP_SGE) { + if (Cond2Op != ICmpInst::ICMP_SLT && Cond1Op != ICmpInst::ICMP_SLE) + return false; + } else if (Cond1Op == ICmpInst::ICMP_SLT || Cond1Op == ICmpInst::ICMP_SLE) { + if (Cond2Op != ICmpInst::ICMP_SGT && Cond1Op != ICmpInst::ICMP_SGE) + return false; + } else { + return false; + } + + PassThroughInfo Info(Cond, BI, 0); + PassThroughs.push_back(Info); + + return true; +} + +// To avoid speculative hoisting certain computations out of +// a basic block. +bool BPFAdjustOpt::avoidSpeculation(Instruction &I) { + if (auto *LdInst = dyn_cast(&I)) { + if (auto *GV = dyn_cast(LdInst->getOperand(0))) { + if (GV->hasAttribute(BPFCoreSharedInfo::AmaAttr) || + GV->hasAttribute(BPFCoreSharedInfo::TypeIdAttr)) + return false; + } + } + + if (!dyn_cast(&I) && !dyn_cast(&I)) + return false; + + // For: + // B1: + // var = ... + // ... + // /* icmp may not be in the same block as var = ... */ + // comp1 = icmp var, ; + // if (comp1) goto B2 else B3; + // B2: + // ... var ... + // change to: + // B1: + // var = ... + // ... + // /* icmp may not be in the same block as var = ... */ + // comp1 = icmp var, ; + // if (comp1) goto B2 else B3; + // B2: + // var = __builtin_bpf_passthrough(seq_num, var); + // ... var ... + bool isCandidate = false; + SmallVector Candidates; + for (User *U : I.users()) { + Instruction *Inst = dyn_cast(U); + if (!Inst) + continue; + + // May cover a little bit more than the + // above pattern. + if (auto *Icmp1 = dyn_cast(Inst)) { + Value *Icmp1Op1 = Icmp1->getOperand(1); + if (!isa(Icmp1Op1)) + return false; + isCandidate = true; + continue; + } + + // Ignore the use in the same basic block as the definition. + if (Inst->getParent() == I.getParent()) + continue; + + // use in a different basic block, If there is a call or + // load/store insn before this instruction in this basic + // block. Most likely it cannot be hoisted out. Skip it. + for (auto &I2 : *Inst->getParent()) { + if (dyn_cast(&I2)) + return false; + if (dyn_cast(&I2) || dyn_cast(&I2)) + return false; + if (&I2 == Inst) + break; + } + + // It should be used in a GEP or a simple arithmetic like + // ZEXT/SEXT which is used for GEP. + if (Inst->getOpcode() == Instruction::ZExt || + Inst->getOpcode() == Instruction::SExt) { + PassThroughInfo Info(&I, Inst, 0); + Candidates.push_back(Info); + } else if (auto *GI = dyn_cast(Inst)) { + // traverse GEP inst to find Use operand index + unsigned i, e; + for (i = 1, e = GI->getNumOperands(); i != e; ++i) { + Value *V = GI->getOperand(i); + if (V == &I) + break; + } + if (i == e) + continue; + + PassThroughInfo Info(&I, GI, i); + Candidates.push_back(Info); + } + } + + if (!isCandidate || Candidates.empty()) + return false; + + PassThroughs.insert(PassThroughs.end(), Candidates.begin(), Candidates.end()); + return true; +} + +void BPFAdjustOpt::adjustBasicBlock(BasicBlock &BB) { + if (!DisableBPFserializeICMP && serializeICMPCrossBB(BB)) + return; +} + +void BPFAdjustOpt::adjustInst(Instruction &I) { + if (!DisableBPFserializeICMP && serializeICMPInBB(I)) + return; + if (!DisableBPFavoidSpeculation && avoidSpeculation(I)) + return; +} diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index e5fc5bac97a8d8..7ef35105083fbc 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -42,6 +42,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() { PassRegistry &PR = *PassRegistry::getPassRegistry(); initializeBPFAbstractMemberAccessLegacyPassPass(PR); initializeBPFPreserveDITypePass(PR); + initializeBPFAdjustOptPass(PR); initializeBPFCheckAndAdjustIRPass(PR); initializeBPFMIPeepholePass(PR); initializeBPFMIPeepholeTruncElimPass(PR); @@ -115,6 +116,11 @@ void BPFTargetMachine::adjustPassManager(PassManagerBuilder &Builder) { PM.add(createCFGSimplificationPass( SimplifyCFGOptions().hoistCommonInsts(true))); }); + Builder.addExtension( + PassManagerBuilder::EP_ModuleOptimizerEarly, + [&](const PassManagerBuilder &, legacy::PassManagerBase &PM) { + PM.add(createBPFAdjustOpt()); + }); } void BPFTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index 27dd685de6bb8a..a17d7173816a89 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -14,6 +14,7 @@ add_public_tablegen_target(BPFCommonTableGen) add_llvm_target(BPFCodeGen BPFAbstractMemberAccess.cpp + BPFAdjustOpt.cpp BPFAsmPrinter.cpp BPFCheckAndAdjustIR.cpp BPFFrameLowering.cpp diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll new file mode 100644 index 00000000000000..bb651f4ea57add --- /dev/null +++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp1.ll @@ -0,0 +1,90 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s +; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s +; +; Source: +; int foo(); +; int bar(int); +; int test() { +; int ret = foo(); +; if (ret <= 0 || ret > 7) +; return 0; +; return bar(ret); +; } +; Compilation flag: +; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c + +; Function Attrs: nounwind +define dso_local i32 @test() #0 { +entry: + %retval = alloca i32, align 4 + %ret = alloca i32, align 4 + %cleanup.dest.slot = alloca i32, align 4 + %0 = bitcast i32* %ret to i8* + call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3 + %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)() + store i32 %call, i32* %ret, align 4, !tbaa !2 + %1 = load i32, i32* %ret, align 4, !tbaa !2 + %cmp = icmp sle i32 %1, 0 + br i1 %cmp, label %if.then, label %lor.lhs.false + +; CHECK: [[REG1:r[0-9]+]] <<= 32 +; CHECK: [[REG1]] s>>= 32 +; CHECK: [[REG2:r[0-9]+]] = 1 +; CHECK: if [[REG2]] s> [[REG1]] goto +; CHECK: if [[REG1]] s> 7 goto + +; CHECK-DISABLE: [[REG1:r[0-9]+]] += -1 +; CHECK-DISABLE: [[REG1]] <<= 32 +; CHECK-DISABLE: [[REG1]] >>= 32 +; CHECK-DISABLE: if [[REG1]] > 6 goto + +lor.lhs.false: ; preds = %entry + %2 = load i32, i32* %ret, align 4, !tbaa !2 + %cmp1 = icmp sgt i32 %2, 7 + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %lor.lhs.false, %entry + store i32 0, i32* %retval, align 4 + store i32 1, i32* %cleanup.dest.slot, align 4 + br label %cleanup + +if.end: ; preds = %lor.lhs.false + %3 = load i32, i32* %ret, align 4, !tbaa !2 + %call2 = call i32 @bar(i32 %3) + store i32 %call2, i32* %retval, align 4 + store i32 1, i32* %cleanup.dest.slot, align 4 + br label %cleanup + +cleanup: ; preds = %if.end, %if.then + %4 = bitcast i32* %ret to i8* + call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #3 + %5 = load i32, i32* %retval, align 4 + ret i32 %5 +} + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1 + +declare dso_local i32 @foo(...) #2 + +declare dso_local i32 @bar(i32) #2 + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1 + +attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { argmemonly nounwind willreturn } +attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll new file mode 100644 index 00000000000000..a264422d2b7627 --- /dev/null +++ b/llvm/test/CodeGen/BPF/adjust-opt-icmp2.ll @@ -0,0 +1,97 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK %s +; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-serialize-icmp %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-DISABLE %s +; +; Source: +; int foo(); +; int bar(int); +; int test() { +; int ret = foo(); +; if (ret <= 0) +; return 0; +; if (ret > 7) +; return 0; +; return bar(ret); +; } +; Compilation flag: +; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c + +; Function Attrs: nounwind +define dso_local i32 @test() #0 { +entry: + %retval = alloca i32, align 4 + %ret = alloca i32, align 4 + %cleanup.dest.slot = alloca i32, align 4 + %0 = bitcast i32* %ret to i8* + call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3 + %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)() + store i32 %call, i32* %ret, align 4, !tbaa !2 + %1 = load i32, i32* %ret, align 4, !tbaa !2 + %cmp = icmp sle i32 %1, 0 + br i1 %cmp, label %if.then, label %if.end + +; CHECK: [[REG1:r[0-9]+]] <<= 32 +; CHECK: [[REG1]] s>>= 32 +; CHECK: [[REG2:r[0-9]+]] = 1 +; CHECK: if [[REG2]] s> [[REG1]] goto +; CHECK: if [[REG1]] s> 7 goto + +; CHECK-DISABLE: [[REG1:r[0-9]+]] += -1 +; CHECK-DISABLE: [[REG1]] <<= 32 +; CHECK-DISABLE: [[REG1]] >>= 32 +; CHECK-DISABLE: if [[REG1]] > 6 goto + +if.then: ; preds = %entry + store i32 0, i32* %retval, align 4 + store i32 1, i32* %cleanup.dest.slot, align 4 + br label %cleanup + +if.end: ; preds = %entry + %2 = load i32, i32* %ret, align 4, !tbaa !2 + %cmp1 = icmp sgt i32 %2, 7 + br i1 %cmp1, label %if.then2, label %if.end3 + +if.then2: ; preds = %if.end + store i32 0, i32* %retval, align 4 + store i32 1, i32* %cleanup.dest.slot, align 4 + br label %cleanup + +if.end3: ; preds = %if.end + %3 = load i32, i32* %ret, align 4, !tbaa !2 + %call4 = call i32 @bar(i32 %3) + store i32 %call4, i32* %retval, align 4 + store i32 1, i32* %cleanup.dest.slot, align 4 + br label %cleanup + +cleanup: ; preds = %if.end3, %if.then2, %if.then + %4 = bitcast i32* %ret to i8* + call void @llvm.lifetime.end.p0i8(i64 4, i8* %4) #3 + %5 = load i32, i32* %retval, align 4 + ret i32 %5 +} + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1 + +declare dso_local i32 @foo(...) #2 + +declare dso_local i32 @bar(i32) #2 + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1 + +attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { argmemonly nounwind willreturn } +attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"int", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll new file mode 100644 index 00000000000000..0d29fff08eed70 --- /dev/null +++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative1.ll @@ -0,0 +1,84 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s +; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s +; +; Source: +; unsigned long foo(); +; void *test(void *p) { +; unsigned long ret = foo(); +; if (ret <= 7) +; p += ret; +; return p; +; } +; Compilation flag: +; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c + +; Function Attrs: nounwind +define dso_local i8* @test(i8* %p) #0 { +entry: + %p.addr = alloca i8*, align 8 + %ret = alloca i64, align 8 + store i8* %p, i8** %p.addr, align 8, !tbaa !2 + %0 = bitcast i64* %ret to i8* + call void @llvm.lifetime.start.p0i8(i64 8, i8* %0) #3 + %call = call i64 bitcast (i64 (...)* @foo to i64 ()*)() + store i64 %call, i64* %ret, align 8, !tbaa !6 + %1 = load i64, i64* %ret, align 8, !tbaa !6 + %cmp = icmp ule i64 %1, 7 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %2 = load i64, i64* %ret, align 8, !tbaa !6 + %3 = load i8*, i8** %p.addr, align 8, !tbaa !2 + %add.ptr = getelementptr i8, i8* %3, i64 %2 + store i8* %add.ptr, i8** %p.addr, align 8, !tbaa !2 + br label %if.end + +if.end: ; preds = %if.then, %entry + %4 = load i8*, i8** %p.addr, align 8, !tbaa !2 + %5 = bitcast i64* %ret to i8* + call void @llvm.lifetime.end.p0i8(i64 8, i8* %5) #3 + ret i8* %4 +} +; CHECK-COMMON: [[REG6:r[0-9]+]] = r1 +; CHECK-COMMON: call foo + +; CHECK: if r0 > 7 goto [[LABEL:.*]] +; CHECK: [[REG6]] += r0 +; CHECK: [[LABEL]]: +; CHECK: r0 = [[REG6]] + +; CHECK-DISABLE: r0 = [[REG6]] +; CHECK-DISABLE: r0 += [[REG1:r[0-9]+]] +; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8 +; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]] +; CHECK-DISABLE: r0 = [[REG6]] +; CHECK-DISABLE: [[LABEL]]: + +; CHECK-COMMON: exit + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1 + +declare dso_local i64 @foo(...) #2 + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1 + +attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { argmemonly nounwind willreturn } +attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"any pointer", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} +!6 = !{!7, !7, i64 0} +!7 = !{!"long", !4, i64 0} diff --git a/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll new file mode 100644 index 00000000000000..3b5cbea9f84b64 --- /dev/null +++ b/llvm/test/CodeGen/BPF/adjust-opt-speculative2.ll @@ -0,0 +1,91 @@ +; RUN: opt -O2 -mtriple=bpf-pc-linux %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK %s +; RUN: opt -O2 -mtriple=bpf-pc-linux -bpf-disable-avoid-speculation %s | llvm-dis > %t1 +; RUN: llc %t1 -o - | FileCheck -check-prefixes=CHECK-COMMON,CHECK-DISABLE %s +; +; Source: +; unsigned foo(); +; void *test(void *p) { +; unsigned ret = foo(); +; if (ret <= 7) +; p += ret; +; return p; +; } +; Compilation flag: +; clang -target bpf -O2 -S -emit-llvm -Xclang -disable-llvm-passes test.c + +; Function Attrs: nounwind +define dso_local i8* @test(i8* %p) #0 { +entry: + %p.addr = alloca i8*, align 8 + %ret = alloca i32, align 4 + store i8* %p, i8** %p.addr, align 8, !tbaa !2 + %0 = bitcast i32* %ret to i8* + call void @llvm.lifetime.start.p0i8(i64 4, i8* %0) #3 + %call = call i32 bitcast (i32 (...)* @foo to i32 ()*)() + store i32 %call, i32* %ret, align 4, !tbaa !6 + %1 = load i32, i32* %ret, align 4, !tbaa !6 + %cmp = icmp ule i32 %1, 7 + br i1 %cmp, label %if.then, label %if.end + +if.then: ; preds = %entry + %2 = load i32, i32* %ret, align 4, !tbaa !6 + %3 = load i8*, i8** %p.addr, align 8, !tbaa !2 + %idx.ext = zext i32 %2 to i64 + %add.ptr = getelementptr i8, i8* %3, i64 %idx.ext + store i8* %add.ptr, i8** %p.addr, align 8, !tbaa !2 + br label %if.end + +if.end: ; preds = %if.then, %entry + %4 = load i8*, i8** %p.addr, align 8, !tbaa !2 + %5 = bitcast i32* %ret to i8* + call void @llvm.lifetime.end.p0i8(i64 4, i8* %5) #3 + ret i8* %4 +} + +; CHECK-COMMON: [[REG6:r[0-9]+]] = r1 +; CHECK-COMMON: call foo + +; CHECK: r0 <<= 32 +; CHECK: r0 >>= 32 +; CHECK: if r0 > 7 goto [[LABEL:.*]] +; CHECK: [[REG6]] += r0 +; CHECK: [[LABEL]]: +; CHECK: r0 = [[REG6]] + +; CHECK-DISABLE: [[REG1:r[0-9]+]] = r0 +; CHECK-DISABLE: [[REG1]] <<= 32 +; CHECK-DISABLE: [[REG1]] >>= 32 +; CHECK-DISABLE: r0 = [[REG6]] +; CHECK-DISABLE: r0 += [[REG1]] +; CHECK-DISABLE: [[REG2:r[0-9]+]] = 8 +; CHECK-DISABLE: if [[REG2]] > [[REG1]] goto [[LABEL:.*]] +; CHECK-DISABLE: r0 = [[REG6]] +; CHECK-DISABLE: [[LABEL]]: + +; CHECK-COMMON: exit + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.start.p0i8(i64 immarg, i8* nocapture) #1 + +declare dso_local i32 @foo(...) #2 + +; Function Attrs: argmemonly nounwind willreturn +declare void @llvm.lifetime.end.p0i8(i64 immarg, i8* nocapture) #1 + +attributes #0 = { nounwind "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "min-legal-vector-width"="0" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #1 = { argmemonly nounwind willreturn } +attributes #2 = { "disable-tail-calls"="false" "frame-pointer"="all" "less-precise-fpmad"="false" "no-infs-fp-math"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "unsafe-fp-math"="false" "use-soft-float"="false" } +attributes #3 = { nounwind } + +!llvm.module.flags = !{!0} +!llvm.ident = !{!1} + +!0 = !{i32 1, !"wchar_size", i32 4} +!1 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git ca9c5433a6c31e372092fcd8bfd0e4fddd7e8784)"} +!2 = !{!3, !3, i64 0} +!3 = !{!"any pointer", !4, i64 0} +!4 = !{!"omnipotent char", !5, i64 0} +!5 = !{!"Simple C/C++ TBAA"} +!6 = !{!7, !7, i64 0} +!7 = !{!"int", !4, i64 0}