From 0e887244bacd26cc19a3c909eccb6ea09025643b Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" Date: Sun, 29 Sep 2024 18:42:20 +0800 Subject: [PATCH] [X86,SimplifyCFG] Allow more PHIs when sinking common code on target supports CMOV --- .../llvm/Analysis/TargetTransformInfo.h | 19 ++- .../llvm/Analysis/TargetTransformInfoImpl.h | 5 +- llvm/lib/Analysis/TargetTransformInfo.cpp | 5 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 6 +- .../lib/Target/X86/X86TargetTransformInfo.cpp | 9 +- llvm/lib/Target/X86/X86TargetTransformInfo.h | 5 +- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 22 ++- .../SimplifyCFG/X86/sink-common-code.ll | 136 ++++++++---------- 8 files changed, 108 insertions(+), 99 deletions(-) diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 89a85bc8a9086..787b471cc328f 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1114,9 +1114,17 @@ class TargetTransformInfo { /// \return the number of registers in the target-provided register class. unsigned getNumberOfRegisters(unsigned ClassID) const; + enum class MoveType { + NoMem = 1, + MemLD = 2, + MemST = 4, + All = 7, + }; + /// \return true if the target supports load/store that enables fault /// suppression of memory operands when the source condition is false. - bool hasConditionalLoadStoreForType(Type *Ty = nullptr) const; + bool hasConditionalMoveForType(Type *Ty = nullptr, + MoveType MT = MoveType::NoMem) const; /// \return the target-provided register class ID for the provided type, /// accounting for type promotion and other type-legalization techniques that @@ -1978,7 +1986,9 @@ class TargetTransformInfo::Concept { virtual bool preferToKeepConstantsAttached(const Instruction &Inst, const Function &Fn) const = 0; virtual unsigned getNumberOfRegisters(unsigned ClassID) const = 0; - virtual bool hasConditionalLoadStoreForType(Type *Ty = nullptr) const = 0; + virtual bool + hasConditionalMoveForType(Type *Ty = nullptr, + MoveType MT = MoveType::NoMem) const = 0; virtual unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const = 0; virtual const char *getRegisterClassName(unsigned ClassID) const = 0; @@ -2571,8 +2581,9 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept { unsigned getNumberOfRegisters(unsigned ClassID) const override { return Impl.getNumberOfRegisters(ClassID); } - bool hasConditionalLoadStoreForType(Type *Ty = nullptr) const override { - return Impl.hasConditionalLoadStoreForType(Ty); + bool hasConditionalMoveForType(Type *Ty = nullptr, + MoveType MT = MoveType::NoMem) const override { + return Impl.hasConditionalMoveForType(Ty, MT); } unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const override { diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index eca8818cc25e6..cf1449b3a7e14 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -470,7 +470,10 @@ class TargetTransformInfoImplBase { } unsigned getNumberOfRegisters(unsigned ClassID) const { return 8; } - bool hasConditionalLoadStoreForType(Type *Ty) const { return false; } + bool hasConditionalMoveForType(Type *Ty, + TargetTransformInfo::MoveType MT) const { + return false; + } unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const { return Vector ? 1 : 0; diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index b5195f764cbd1..231731b2e7446 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -727,8 +727,9 @@ unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const { return TTIImpl->getNumberOfRegisters(ClassID); } -bool TargetTransformInfo::hasConditionalLoadStoreForType(Type *Ty) const { - return TTIImpl->hasConditionalLoadStoreForType(Ty); +bool TargetTransformInfo::hasConditionalMoveForType(Type *Ty, + MoveType MT) const { + return TTIImpl->hasConditionalMoveForType(Ty, MT); } unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 25213f587116d..5c35142c951d6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -4806,7 +4806,8 @@ void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, TLI.getTargetMachine().getTargetTransformInfo(*I.getFunction()); SDValue StoreNode = !IsCompressing && - TTI.hasConditionalLoadStoreForType(I.getArgOperand(0)->getType()) + TTI.hasConditionalMoveForType(I.getArgOperand(0)->getType(), + TargetTransformInfo::MoveType::All) ? TLI.visitMaskedStore(DAG, sdl, getMemoryRoot(), MMO, Ptr, Src0, Mask) : DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, @@ -4992,7 +4993,8 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { SDValue Load; SDValue Res; if (!IsExpanding && - TTI.hasConditionalLoadStoreForType(Src0Operand->getType())) + TTI.hasConditionalMoveForType(Src0Operand->getType(), + TargetTransformInfo::MoveType::All)) Res = TLI.visitMaskedLoad(DAG, sdl, InChain, MMO, Load, Ptr, Src0, Mask); else Res = Load = diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 46bc73c5e928e..6c213bd8eb696 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -176,8 +176,11 @@ unsigned X86TTIImpl::getNumberOfRegisters(unsigned ClassID) const { return 8; } -bool X86TTIImpl::hasConditionalLoadStoreForType(Type *Ty) const { - if (!ST->hasCF()) +bool X86TTIImpl::hasConditionalMoveForType( + Type *Ty, TargetTransformInfo::MoveType MT) const { + if (!ST->canUseCMOV()) + return false; + if (MT != TargetTransformInfo::MoveType::NoMem && !ST->hasCF()) return false; if (!Ty) return true; @@ -6064,7 +6067,7 @@ bool X86TTIImpl::isLegalMaskedLoad(Type *DataTy, Align Alignment) { // The backend can't handle a single element vector w/o CFCMOV. if (isa(DataTy) && cast(DataTy)->getNumElements() == 1) - return ST->hasCF() && hasConditionalLoadStoreForType(ScalarTy); + return ST->hasCF() && hasConditionalMoveForType(ScalarTy); if (!ST->hasAVX()) return false; diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index c16461b157e07..4515f29bd2a6e 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -132,7 +132,10 @@ class X86TTIImpl : public BasicTTIImplBase { /// @{ unsigned getNumberOfRegisters(unsigned ClassID) const; - bool hasConditionalLoadStoreForType(Type *Ty = nullptr) const; + bool + hasConditionalMoveForType(Type *Ty = nullptr, + TargetTransformInfo::MoveType MT = + TargetTransformInfo::MoveType::NoMem) const; TypeSize getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const; unsigned getLoadStoreVecRegBitWidth(unsigned AS) const; unsigned getMaxInterleaveFactor(ElementCount VF); diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 1f2c9389c008b..800ea07c61929 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -129,6 +129,10 @@ static cl::opt HoistLoadsStoresWithCondFaultingThreshold( "to speculatively execute to eliminate conditional branch " "(default = 6)")); +static cl::opt ProfitableToSinkInstructionThreshold( + "profitable-to-sink-instruction-threshold", cl::Hidden, cl::init(6), + cl::desc("Control the maximal PHI instructions")); + static cl::opt HoistCommonSkipLimit("simplifycfg-hoist-common-skip-limit", cl::Hidden, cl::init(20), @@ -1742,7 +1746,8 @@ static bool isSafeCheapLoadStore(const Instruction *I, // llvm.masked.load/store use i32 for alignment while load/store use i64. // That's why we have the alignment limitation. // FIXME: Update the prototype of the intrinsics? - return TTI.hasConditionalLoadStoreForType(getLoadStoreType(I)) && + return TTI.hasConditionalMoveForType(getLoadStoreType(I), + TargetTransformInfo::MoveType::All) && getLoadStoreAlignment(I) < Value::MaximumAlignment; } @@ -2386,8 +2391,8 @@ namespace { /// Check whether BB's predecessors end with unconditional branches. If it is /// true, sink any common code from the predecessors to BB. -static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, - DomTreeUpdater *DTU) { +static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, DomTreeUpdater *DTU, + const TargetTransformInfo &TTI) { // We support two situations: // (1) all incoming arcs are unconditional // (2) there are non-unconditional incoming arcs @@ -2492,12 +2497,16 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, // sink? auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) { unsigned NumPHIInsts = 0; + unsigned NumCmovInsts = 0; for (Use &U : (*LRI)[0]->operands()) { auto It = PHIOperands.find(&U); if (It != PHIOperands.end() && !all_of(It->second, [&](Value *V) { return InstructionsToSink.contains(V); })) { - ++NumPHIInsts; + if (TTI.hasConditionalMoveForType(U->getType())) + ++NumCmovInsts; + else + ++NumPHIInsts; // Do not separate a load/store from the gep producing the address. // The gep can likely be folded into the load/store as an addressing // mode. Additionally, a load of a gep is easier to analyze than a @@ -2511,7 +2520,8 @@ static bool sinkCommonCodeFromPredecessors(BasicBlock *BB, } } LLVM_DEBUG(dbgs() << "SINK: #phi insts: " << NumPHIInsts << "\n"); - return NumPHIInsts <= 1; + return NumPHIInsts <= 1 && + NumCmovInsts < ProfitableToSinkInstructionThreshold; }; // We've determined that we are going to sink last ScanIdx instructions, @@ -8119,7 +8129,7 @@ bool SimplifyCFGOpt::simplifyOnce(BasicBlock *BB) { return true; if (SinkCommon && Options.SinkCommonInsts) - if (sinkCommonCodeFromPredecessors(BB, DTU) || + if (sinkCommonCodeFromPredecessors(BB, DTU, TTI) || mergeCompatibleInvokes(BB, DTU)) { // sinkCommonCodeFromPredecessors() does not automatically CSE PHI's, // so we may now how duplicate PHI's. diff --git a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll index 170f8d1592c2f..912ba0d8c8f3a 100644 --- a/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/sink-common-code.ll @@ -220,17 +220,15 @@ define i32 @test8(i1 zeroext %flag, i32 %x, ptr %y) { ; CHECK-LABEL: @test8( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[Z:%.*]] = load volatile i32, ptr [[Y:%.*]], align 4 -; CHECK-NEXT: br i1 [[FLAG:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then: -; CHECK-NEXT: [[A:%.*]] = add i32 [[Z]], 5 -; CHECK-NEXT: br label [[IF_END:%.*]] +; CHECK-NEXT: br i1 [[FLAG:%.*]], label [[IF_END:%.*]], label [[IF_ELSE:%.*]] ; CHECK: if.else: ; CHECK-NEXT: [[W:%.*]] = load volatile i32, ptr [[Y]], align 4 -; CHECK-NEXT: [[B:%.*]] = add i32 [[W]], 7 ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[B_SINK:%.*]] = phi i32 [ [[B]], [[IF_ELSE]] ], [ [[A]], [[IF_THEN]] ] -; CHECK-NEXT: store volatile i32 [[B_SINK]], ptr [[Y]], align 4 +; CHECK-NEXT: [[DOTSINK:%.*]] = phi i32 [ 7, [[IF_ELSE]] ], [ 5, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[W_SINK:%.*]] = phi i32 [ [[W]], [[IF_ELSE]] ], [ [[Z]], [[ENTRY]] ] +; CHECK-NEXT: [[B:%.*]] = add i32 [[W_SINK]], [[DOTSINK]] +; CHECK-NEXT: store volatile i32 [[B]], ptr [[Y]], align 4 ; CHECK-NEXT: ret i32 1 ; entry: @@ -262,15 +260,15 @@ define i32 @test9(i1 zeroext %flag, i32 %x, ptr %y, ptr %p) { ; CHECK-NEXT: store i32 7, ptr [[P:%.*]], align 4 ; CHECK-NEXT: [[Z:%.*]] = load volatile i32, ptr [[Y:%.*]], align 4 ; CHECK-NEXT: store i32 6, ptr [[P]], align 4 -; CHECK-NEXT: [[A:%.*]] = add i32 [[Z]], 5 ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: ; CHECK-NEXT: [[W:%.*]] = load volatile i32, ptr [[Y]], align 4 -; CHECK-NEXT: [[B:%.*]] = add i32 [[W]], 7 ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[B_SINK:%.*]] = phi i32 [ [[B]], [[IF_ELSE]] ], [ [[A]], [[IF_THEN]] ] -; CHECK-NEXT: store volatile i32 [[B_SINK]], ptr [[Y]], align 4 +; CHECK-NEXT: [[DOTSINK:%.*]] = phi i32 [ 7, [[IF_ELSE]] ], [ 5, [[IF_THEN]] ] +; CHECK-NEXT: [[W_SINK:%.*]] = phi i32 [ [[W]], [[IF_ELSE]] ], [ [[Z]], [[IF_THEN]] ] +; CHECK-NEXT: [[B:%.*]] = add i32 [[W_SINK]], [[DOTSINK]] +; CHECK-NEXT: store volatile i32 [[B]], ptr [[Y]], align 4 ; CHECK-NEXT: ret i32 1 ; entry: @@ -570,16 +568,16 @@ define zeroext i1 @test_crash(i1 zeroext %flag, ptr %i4, ptr %m, ptr %n) { ; CHECK-NEXT: br i1 [[FLAG:%.*]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]] ; CHECK: if.then: ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[I4:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1 ; CHECK-NEXT: br label [[IF_END:%.*]] ; CHECK: if.else: ; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[M:%.*]], align 4 ; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[N:%.*]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3]], [[TMP4]] ; CHECK-NEXT: br label [[IF_END]] ; CHECK: if.end: -; CHECK-NEXT: [[TMP5_SINK:%.*]] = phi i32 [ [[TMP5]], [[IF_ELSE]] ], [ [[TMP2]], [[IF_THEN]] ] -; CHECK-NEXT: store i32 [[TMP5_SINK]], ptr [[I4]], align 4 +; CHECK-NEXT: [[TMP4_SINK:%.*]] = phi i32 [ [[TMP4]], [[IF_ELSE]] ], [ -1, [[IF_THEN]] ] +; CHECK-NEXT: [[TMP3_SINK:%.*]] = phi i32 [ [[TMP3]], [[IF_ELSE]] ], [ [[TMP1]], [[IF_THEN]] ] +; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[TMP3_SINK]], [[TMP4_SINK]] +; CHECK-NEXT: store i32 [[TMP5]], ptr [[I4]], align 4 ; CHECK-NEXT: ret i1 true ; entry: @@ -1475,22 +1473,14 @@ declare void @llvm.lifetime.end.p0(i64, ptr nocapture) define void @creating_too_many_phis(i1 %cond, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e, i32 %f, i32 %g, i32 %h) { ; CHECK-LABEL: @creating_too_many_phis( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[BB0:%.*]], label [[BB1:%.*]] -; CHECK: bb0: -; CHECK-NEXT: [[V0:%.*]] = add i32 [[A:%.*]], [[B:%.*]] -; CHECK-NEXT: [[V1:%.*]] = add i32 [[V0]], [[C:%.*]] -; CHECK-NEXT: [[V2:%.*]] = add i32 [[D:%.*]], [[E:%.*]] -; CHECK-NEXT: [[R3:%.*]] = add i32 [[V1]], [[V2]] -; CHECK-NEXT: br label [[END:%.*]] -; CHECK: bb1: -; CHECK-NEXT: [[V4:%.*]] = add i32 [[A]], [[B]] -; CHECK-NEXT: [[V5:%.*]] = add i32 [[V4]], [[C]] -; CHECK-NEXT: [[V6:%.*]] = add i32 [[G:%.*]], [[H:%.*]] +; CHECK-NEXT: end: +; CHECK-NEXT: [[E_H:%.*]] = select i1 [[COND:%.*]], i32 [[E:%.*]], i32 [[H:%.*]] +; CHECK-NEXT: [[D_G:%.*]] = select i1 [[COND]], i32 [[D:%.*]], i32 [[G:%.*]] +; CHECK-NEXT: [[V4:%.*]] = add i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[V5:%.*]] = add i32 [[V4]], [[C:%.*]] +; CHECK-NEXT: [[V6:%.*]] = add i32 [[D_G]], [[E_H]] ; CHECK-NEXT: [[R7:%.*]] = add i32 [[V5]], [[V6]] -; CHECK-NEXT: br label [[END]] -; CHECK: end: -; CHECK-NEXT: [[R7_SINK:%.*]] = phi i32 [ [[R7]], [[BB1]] ], [ [[R3]], [[BB0]] ] -; CHECK-NEXT: call void @use32(i32 [[R7_SINK]]) +; CHECK-NEXT: call void @use32(i32 [[R7]]) ; CHECK-NEXT: ret void ; br i1 %cond, label %bb0, label %bb1 @@ -1806,17 +1796,17 @@ define i64 @multi_use_in_block_inconsistent(i1 %cond, ptr %p, i64 %a, i64 %b) { ; CHECK-NEXT: call void @dummy() ; CHECK-NEXT: [[GEP1_A:%.*]] = getelementptr i8, ptr [[P:%.*]], i64 [[A:%.*]] ; CHECK-NEXT: [[V_A:%.*]] = load i64, ptr [[GEP1_A]], align 8 -; CHECK-NEXT: [[GEP2_A:%.*]] = getelementptr i8, ptr [[GEP1_A]], i64 [[V_A]] ; CHECK-NEXT: br label [[JOIN:%.*]] ; CHECK: else: ; CHECK-NEXT: [[GEP1_B:%.*]] = getelementptr i8, ptr [[P]], i64 [[A]] ; CHECK-NEXT: [[V_B:%.*]] = load i64, ptr [[P]], align 8 -; CHECK-NEXT: [[GEP2_B:%.*]] = getelementptr i8, ptr [[GEP1_B]], i64 [[V_B]] ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: +; CHECK-NEXT: [[V_B_SINK:%.*]] = phi i64 [ [[V_B]], [[ELSE]] ], [ [[V_A]], [[IF]] ] +; CHECK-NEXT: [[GEP1_B_SINK:%.*]] = phi ptr [ [[GEP1_B]], [[ELSE]] ], [ [[GEP1_A]], [[IF]] ] ; CHECK-NEXT: [[PHI1:%.*]] = phi i64 [ [[V_A]], [[IF]] ], [ [[V_B]], [[ELSE]] ] -; CHECK-NEXT: [[PHI2:%.*]] = phi ptr [ [[GEP2_A]], [[IF]] ], [ [[GEP2_B]], [[ELSE]] ] -; CHECK-NEXT: call void @use.ptr(ptr [[PHI2]]) +; CHECK-NEXT: [[GEP2_B:%.*]] = getelementptr i8, ptr [[GEP1_B_SINK]], i64 [[V_B_SINK]] +; CHECK-NEXT: call void @use.ptr(ptr [[GEP2_B]]) ; CHECK-NEXT: ret i64 [[PHI1]] ; br i1 %cond, label %if, label %else @@ -1873,19 +1863,16 @@ join: define i64 @load_with_non_sunk_gep_both(i1 %cond, ptr %p.a, ptr %p.b, i64 %a, i64 %b) { ; CHECK-LABEL: @load_with_non_sunk_gep_both( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]] ; CHECK: if: ; CHECK-NEXT: call void @dummy() -; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i8, ptr [[P_A:%.*]], i64 [[A:%.*]] -; CHECK-NEXT: [[V_A:%.*]] = load i64, ptr [[GEP_A]], align 8 -; CHECK-NEXT: br label [[JOIN:%.*]] -; CHECK: else: -; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i8, ptr [[P_B:%.*]], i64 [[B:%.*]] -; CHECK-NEXT: [[V_B:%.*]] = load i64, ptr [[GEP_B]], align 8 ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: -; CHECK-NEXT: [[V:%.*]] = phi i64 [ [[V_A]], [[IF]] ], [ [[V_B]], [[ELSE]] ] -; CHECK-NEXT: ret i64 [[V]] +; CHECK-NEXT: [[B_SINK:%.*]] = phi i64 [ [[A:%.*]], [[IF]] ], [ [[B:%.*]], [[TMP0:%.*]] ] +; CHECK-NEXT: [[P_B_SINK:%.*]] = phi ptr [ [[P_A:%.*]], [[IF]] ], [ [[P_B:%.*]], [[TMP0]] ] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i8, ptr [[P_B_SINK]], i64 [[B_SINK]] +; CHECK-NEXT: [[V_B:%.*]] = load i64, ptr [[GEP_B]], align 8 +; CHECK-NEXT: ret i64 [[V_B]] ; br i1 %cond, label %if, label %else @@ -1971,17 +1958,15 @@ join: define void @store_with_non_sunk_gep(i1 %cond, ptr %p.a, ptr %p.b, i64 %a, i64 %b) { ; CHECK-LABEL: @store_with_non_sunk_gep( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]] ; CHECK: if: ; CHECK-NEXT: call void @dummy() -; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i8, ptr [[P_A:%.*]], i64 [[A:%.*]] -; CHECK-NEXT: store i64 0, ptr [[GEP_A]], align 8 -; CHECK-NEXT: br label [[JOIN:%.*]] -; CHECK: else: -; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i8, ptr [[P_B:%.*]], i64 [[B:%.*]] -; CHECK-NEXT: store i64 0, ptr [[GEP_B]], align 8 ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: +; CHECK-NEXT: [[B_SINK:%.*]] = phi i64 [ [[A:%.*]], [[IF]] ], [ [[B:%.*]], [[TMP0:%.*]] ] +; CHECK-NEXT: [[P_B_SINK:%.*]] = phi ptr [ [[P_A:%.*]], [[IF]] ], [ [[P_B:%.*]], [[TMP0]] ] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i8, ptr [[P_B_SINK]], i64 [[B_SINK]] +; CHECK-NEXT: store i64 0, ptr [[GEP_B]], align 8 ; CHECK-NEXT: ret void ; br i1 %cond, label %if, label %else @@ -2003,17 +1988,15 @@ join: define void @store_with_non_sunk_gep_as_value(i1 %cond, ptr %p, ptr %p.a, ptr %p.b, i64 %a, i64 %b) { ; CHECK-LABEL: @store_with_non_sunk_gep_as_value( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]] ; CHECK: if: ; CHECK-NEXT: call void @dummy() -; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i8, ptr [[P_A:%.*]], i64 [[A:%.*]] -; CHECK-NEXT: br label [[JOIN:%.*]] -; CHECK: else: -; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i8, ptr [[P_B:%.*]], i64 [[B:%.*]] ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: -; CHECK-NEXT: [[GEP_B_SINK:%.*]] = phi ptr [ [[GEP_B]], [[ELSE]] ], [ [[GEP_A]], [[IF]] ] -; CHECK-NEXT: store ptr [[GEP_B_SINK]], ptr [[P:%.*]], align 8 +; CHECK-NEXT: [[B_SINK:%.*]] = phi i64 [ [[A:%.*]], [[IF]] ], [ [[B:%.*]], [[TMP0:%.*]] ] +; CHECK-NEXT: [[P_B_SINK:%.*]] = phi ptr [ [[P_A:%.*]], [[IF]] ], [ [[P_B:%.*]], [[TMP0]] ] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr i8, ptr [[P_B_SINK]], i64 [[B_SINK]] +; CHECK-NEXT: store ptr [[GEP_B]], ptr [[P:%.*]], align 8 ; CHECK-NEXT: ret void ; br i1 %cond, label %if, label %else @@ -2078,18 +2061,16 @@ join: define i32 @store_and_unrelated_many_phi_add(i1 %cond, ptr %p, i32 %a, i32 %b) { ; CHECK-LABEL: @store_and_unrelated_many_phi_add( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]] ; CHECK: if: ; CHECK-NEXT: call void @dummy() -; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[A:%.*]], 2 -; CHECK-NEXT: br label [[JOIN:%.*]] -; CHECK: else: -; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[B:%.*]], 3 ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: -; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[ADD_1]], [[IF]] ], [ [[ADD_2]], [[ELSE]] ] +; CHECK-NEXT: [[DOTSINK:%.*]] = phi i32 [ 2, [[IF]] ], [ 3, [[TMP0:%.*]] ] +; CHECK-NEXT: [[B_SINK:%.*]] = phi i32 [ [[A:%.*]], [[IF]] ], [ [[B:%.*]], [[TMP0]] ] +; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[B_SINK]], [[DOTSINK]] ; CHECK-NEXT: store i32 1, ptr [[P:%.*]], align 4 -; CHECK-NEXT: ret i32 [[PHI]] +; CHECK-NEXT: ret i32 [[ADD_2]] ; br i1 %cond, label %if, label %else @@ -2111,19 +2092,16 @@ join: define i32 @store_and_related_many_phi_add(i1 %cond, ptr %p, i32 %a, i32 %b) { ; CHECK-LABEL: @store_and_related_many_phi_add( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]] ; CHECK: if: ; CHECK-NEXT: call void @dummy() -; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[A:%.*]], 2 -; CHECK-NEXT: br label [[JOIN:%.*]] -; CHECK: else: -; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[B:%.*]], 3 ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: -; CHECK-NEXT: [[ADD_2_SINK:%.*]] = phi i32 [ [[ADD_2]], [[ELSE]] ], [ [[ADD_1]], [[IF]] ] -; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[ADD_1]], [[IF]] ], [ [[ADD_2]], [[ELSE]] ] -; CHECK-NEXT: store i32 [[ADD_2_SINK]], ptr [[P:%.*]], align 4 -; CHECK-NEXT: ret i32 [[PHI]] +; CHECK-NEXT: [[DOTSINK:%.*]] = phi i32 [ 2, [[IF]] ], [ 3, [[TMP0:%.*]] ] +; CHECK-NEXT: [[B_SINK:%.*]] = phi i32 [ [[A:%.*]], [[IF]] ], [ [[B:%.*]], [[TMP0]] ] +; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[B_SINK]], [[DOTSINK]] +; CHECK-NEXT: store i32 [[ADD_2]], ptr [[P:%.*]], align 4 +; CHECK-NEXT: ret i32 [[ADD_2]] ; br i1 %cond, label %if, label %else @@ -2145,19 +2123,17 @@ join: define i32 @store_and_unrelated_many_phi_add2(i1 %cond, ptr %p, i32 %a, i32 %b) { ; CHECK-LABEL: @store_and_unrelated_many_phi_add2( -; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[ELSE:%.*]] +; CHECK-NEXT: br i1 [[COND:%.*]], label [[IF:%.*]], label [[JOIN:%.*]] ; CHECK: if: ; CHECK-NEXT: call void @dummy() -; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[A:%.*]], 2 -; CHECK-NEXT: br label [[JOIN:%.*]] -; CHECK: else: -; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[B:%.*]], 3 ; CHECK-NEXT: br label [[JOIN]] ; CHECK: join: -; CHECK-NEXT: [[PHI:%.*]] = phi i32 [ [[ADD_1]], [[IF]] ], [ [[ADD_2]], [[ELSE]] ] +; CHECK-NEXT: [[DOTSINK:%.*]] = phi i32 [ 2, [[IF]] ], [ 3, [[TMP0:%.*]] ] +; CHECK-NEXT: [[B_SINK:%.*]] = phi i32 [ [[A:%.*]], [[IF]] ], [ [[B:%.*]], [[TMP0]] ] +; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[B_SINK]], [[DOTSINK]] ; CHECK-NEXT: [[ADD_A_2:%.*]] = add i32 [[A]], 1 ; CHECK-NEXT: store i32 [[ADD_A_2]], ptr [[P:%.*]], align 4 -; CHECK-NEXT: ret i32 [[PHI]] +; CHECK-NEXT: ret i32 [[ADD_2]] ; br i1 %cond, label %if, label %else