diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index 1c841155e6434..f0418a8efe398 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -105,6 +105,9 @@ class MachineInstr // known to be exact. NoFPExcept = 1 << 14, // Instruction does not raise // floatint-point exceptions. + NoMerge = 1 << 15, // Passes that drop source location info + // (e.g. branch folding) should skip + // this instruction. }; private: diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 462d9f91c4f15..590919f89cac7 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -278,6 +278,7 @@ class SelectionDAG { struct CallSiteDbgInfo { CallSiteInfo CSInfo; MDNode *HeapAllocSite = nullptr; + bool NoMerge = false; }; DenseMap SDCallSiteDbgInfo; @@ -1916,6 +1917,18 @@ class SelectionDAG { return It->second.HeapAllocSite; } + void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge) { + if (NoMerge) + SDCallSiteDbgInfo[Node].NoMerge = NoMerge; + } + + bool getNoMergeSiteInfo(const SDNode *Node) { + auto I = SDCallSiteDbgInfo.find(Node); + if (I == SDCallSiteDbgInfo.end()) + return false; + return I->second.NoMerge; + } + /// Return the current function's default denormal handling kind for the given /// floating point type. DenormalMode getDenormalMode(EVT VT) const { diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 2689838b3e7cc..d2fb5afb9adbf 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -3613,6 +3613,7 @@ class TargetLowering : public TargetLoweringBase { bool IsConvergent : 1; bool IsPatchPoint : 1; bool IsPreallocated : 1; + bool NoMerge : 1; // IsTailCall should be modified by implementations of // TargetLowering::LowerCall that perform tail call conversions. @@ -3636,7 +3637,8 @@ class TargetLowering : public TargetLoweringBase { CallLoweringInfo(SelectionDAG &DAG) : RetSExt(false), RetZExt(false), IsVarArg(false), IsInReg(false), DoesNotReturn(false), IsReturnValueUsed(true), IsConvergent(false), - IsPatchPoint(false), IsPreallocated(false), DAG(DAG) {} + IsPatchPoint(false), IsPreallocated(false), NoMerge(false), + DAG(DAG) {} CallLoweringInfo &setDebugLoc(const SDLoc &dl) { DL = dl; @@ -3685,7 +3687,8 @@ class TargetLowering : public TargetLoweringBase { IsReturnValueUsed = !Call.use_empty(); RetSExt = Call.hasRetAttr(Attribute::SExt); RetZExt = Call.hasRetAttr(Attribute::ZExt); - + NoMerge = Call.hasFnAttr(Attribute::NoMerge); + Callee = Target; CallConv = Call.getCallingConv(); diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp index 852bfb36c7208..df79019a6402d 100644 --- a/llvm/lib/CodeGen/BranchFolding.cpp +++ b/llvm/lib/CodeGen/BranchFolding.cpp @@ -348,6 +348,9 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1, MBBI1->isInlineAsm()) { break; } + if (MBBI1->getFlag(MachineInstr::NoMerge) || + MBBI2->getFlag(MachineInstr::NoMerge)) + break; ++TailLen; I1 = MBBI1; I2 = MBBI2; diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 36cd39f2c4759..fa23df6288e99 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -778,6 +778,8 @@ void MIPrinter::print(const MachineInstr &MI) { OS << "exact "; if (MI.getFlag(MachineInstr::NoFPExcept)) OS << "nofpexcept "; + if (MI.getFlag(MachineInstr::NoMerge)) + OS << "nomerge "; OS << TII->getName(MI.getOpcode()); if (I < E) diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 7afa61f2c4dbd..987de0cb3b0e5 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -1595,6 +1595,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << "exact "; if (getFlag(MachineInstr::NoFPExcept)) OS << "nofpexcept "; + if (getFlag(MachineInstr::NoMerge)) + OS << "nomerge "; // Print the opcode name. if (TII) diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index 83aaf93c9ba0d..731cd2396b626 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -872,6 +872,10 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { DAG->getTarget().Options.EmitCallSiteInfo) MF.addCallArgsForwardingRegs(MI, DAG->getSDCallSiteInfo(Node)); + if (DAG->getNoMergeSiteInfo(Node)) { + MI->setFlag(MachineInstr::MIFlag::NoMerge); + } + return MI; }; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 81b50346437c7..aece1d0da59ab 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -4541,6 +4541,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, // Returns a chain and a flag for retval copy to use. Chain = DAG.getNode(AArch64ISD::CALL, DL, NodeTys, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); InFlag = Chain.getValue(1); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 98161c3494445..bb50525b673b7 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2542,6 +2542,7 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, // Returns a chain and a flag for retval copy to use. Chain = DAG.getNode(CallOpc, dl, NodeTys, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); InFlag = Chain.getValue(1); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 53f9ac678c7b7..7e0cbbff2515c 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -5586,6 +5586,7 @@ SDValue PPCTargetLowering::FinishCall( std::array ReturnTypes = {{MVT::Other, MVT::Glue}}; Chain = DAG.getNode(CallOpc, dl, ReturnTypes, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CFlags.NoMerge); Glue = Chain.getValue(1); // When performing tail call optimization the callee pops its arguments off @@ -5667,7 +5668,8 @@ PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, isIndirectCall(Callee, DAG, Subtarget, isPatchPoint), // hasNest Subtarget.is64BitELFABI() && - any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); })); + any_of(Outs, [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }), + CLI.NoMerge); if (Subtarget.isSVR4ABI() && Subtarget.isPPC64()) return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG, diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index 29d4e54edc672..77083b4761655 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -979,12 +979,13 @@ namespace llvm { const bool IsPatchPoint : 1; const bool IsIndirect : 1; const bool HasNest : 1; + const bool NoMerge : 1; CallFlags(CallingConv::ID CC, bool IsTailCall, bool IsVarArg, - bool IsPatchPoint, bool IsIndirect, bool HasNest) + bool IsPatchPoint, bool IsIndirect, bool HasNest, bool NoMerge) : CallConv(CC), IsTailCall(IsTailCall), IsVarArg(IsVarArg), IsPatchPoint(IsPatchPoint), IsIndirect(IsIndirect), - HasNest(HasNest) {} + HasNest(HasNest), NoMerge(NoMerge) {} }; private: diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 8035d42e1d420..7b1174491e64e 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -2353,6 +2353,7 @@ SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI, } Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); Glue = Chain.getValue(1); // Mark the end of the call, which is glued to the call itself. diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index a1121600346f2..3d2cdccd50a59 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -4336,6 +4336,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops); } InFlag = Chain.getValue(1); + DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge); DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo)); // Save heapallocsite metadata. diff --git a/llvm/test/CodeGen/AArch64/nomerge.ll b/llvm/test/CodeGen/AArch64/nomerge.ll new file mode 100644 index 0000000000000..4ef1027560802 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/nomerge.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=aarch64 -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: // %bb.0: // %entry +; CHECK: // %bb.1: // %entry +; CHECK: // %bb.2: // %if.then +; CHECK-NEXT: bl bar +; CHECK: b bar +; CHECK: .LBB0_3: // %if.then2 +; CHECK-NEXT: bl bar +; CHECK: .LBB0_4: // %if.end3 +; CHECK: b bar diff --git a/llvm/test/CodeGen/ARM/nomerge.ll b/llvm/test/CodeGen/ARM/nomerge.ll new file mode 100644 index 0000000000000..b4e01c0560cf7 --- /dev/null +++ b/llvm/test/CodeGen/ARM/nomerge.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=arm -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: @ %bb.0: @ %entry +; CHECK: @ %bb.1: @ %entry +; CHECK: @ %bb.2: @ %if.then +; CHECK-NEXT: bl bar +; CHECK: b bar +; CHECK: .LBB0_3: @ %if.then2 +; CHECK-NEXT: bl bar +; CHECK: .LBB0_4: @ %if.end3 +; CHECK: b bar diff --git a/llvm/test/CodeGen/PowerPC/nomerge.ll b/llvm/test/CodeGen/PowerPC/nomerge.ll new file mode 100644 index 0000000000000..4e3db233e3ce0 --- /dev/null +++ b/llvm/test/CodeGen/PowerPC/nomerge.ll @@ -0,0 +1,35 @@ +; RUN: llc < %s -mtriple=powerpc -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK: # %bb.1: # %entry +; CHECK: # %bb.2: # %if.then +; CHECK-NEXT: bl bar +; CHECK: .LBB0_3: # %if.then2 +; CHECK-NEXT: bl bar +; CHECK: .LBB0_4: # %if.end3 +; CHECK-NEXT: bl bar diff --git a/llvm/test/CodeGen/RISCV/nomerge.ll b/llvm/test/CodeGen/RISCV/nomerge.ll new file mode 100644 index 0000000000000..c35c708d0fcbf --- /dev/null +++ b/llvm/test/CodeGen/RISCV/nomerge.ll @@ -0,0 +1,35 @@ +; RUN: llc < %s -mtriple=riscv64 -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK: # %bb.1: # %entry +; CHECK: # %bb.2: # %if.then +; CHECK-NEXT: call bar +; CHECK: .LBB0_3: # %if.then2 +; CHECK-NEXT: call bar +; CHECK: .LBB0_4: # %if.end3 +; CHECK: tail bar diff --git a/llvm/test/CodeGen/X86/nomerge.ll b/llvm/test/CodeGen/X86/nomerge.ll new file mode 100644 index 0000000000000..8da27f79db908 --- /dev/null +++ b/llvm/test/CodeGen/X86/nomerge.ll @@ -0,0 +1,36 @@ +; RUN: llc < %s -mtriple=x86_64 -o - | FileCheck %s + +define void @foo(i32 %i) { +entry: + switch i32 %i, label %if.end3 [ + i32 5, label %if.then + i32 7, label %if.then2 + ] + +if.then: + tail call void @bar() #0 + br label %if.end3 + +if.then2: + tail call void @bar() #0 + br label %if.end3 + +if.end3: + tail call void @bar() #0 + ret void +} + +declare void @bar() + +attributes #0 = { nomerge } + +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK: # %bb.1: # %entry +; CHECK: # %bb.2: # %if.then +; CHECK-NEXT: callq bar +; CHECK: jmp bar # TAILCALL +; CHECK: .LBB0_3: # %if.then2 +; CHECK: callq bar +; CHECK: .LBB0_4: # %if.end3 +; CHECK: jmp bar # TAILCALL