diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index ce5aea2e8d34d..4d81bb692b47c 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -416,6 +416,10 @@ class Value; /// Return true if the only users of this pointer are lifetime markers. bool onlyUsedByLifetimeMarkers(const Value *V); + /// Return true if the only users of this pointer are lifetime markers or + /// droppable instructions. + bool onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V); + /// Return true if speculation of the given load must be suppressed to avoid /// ordering or interfering with an active sanitizer. If not suppressed, /// dereferenceability and alignment must be proven separately. Note: This diff --git a/llvm/include/llvm/IR/Value.h b/llvm/include/llvm/IR/Value.h index 04ca682746264..fa706a1b64086 100644 --- a/llvm/include/llvm/IR/Value.h +++ b/llvm/include/llvm/IR/Value.h @@ -470,6 +470,9 @@ class Value { void dropDroppableUses(llvm::function_ref ShouldDrop = [](const Use *) { return true; }); + /// Remove every use of \p User that can safely be removed. + void dropDroppableUsesByUser(const User &Usr); + /// Check if this value is used in the specified basic block. bool isUsedInBasicBlock(const BasicBlock *BB) const; diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 116916a9be2d2..271200f7030a2 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -4302,18 +4302,33 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V, return true; } -/// Return true if the only users of this pointer are lifetime markers. -bool llvm::onlyUsedByLifetimeMarkers(const Value *V) { +static bool onlyUsedByLifetimeMarkersOrDroppableInstsHelper( + const Value *V, bool AllowLifetime, bool AllowDroppable) { for (const User *U : V->users()) { const IntrinsicInst *II = dyn_cast(U); - if (!II) return false; - - if (!II->isLifetimeStartOrEnd()) + if (!II) return false; + + if (AllowLifetime && II->isLifetimeStartOrEnd()) + continue; + + if (AllowDroppable && II->isDroppable()) + continue; + + return false; } return true; } +bool llvm::onlyUsedByLifetimeMarkers(const Value *V) { + return onlyUsedByLifetimeMarkersOrDroppableInstsHelper( + V, /* AllowLifetime */ true, /* AllowDroppable */ false); +} +bool llvm::onlyUsedByLifetimeMarkersOrDroppableInsts(const Value *V) { + return onlyUsedByLifetimeMarkersOrDroppableInstsHelper( + V, /* AllowLifetime */ true, /* AllowDroppable */ true); +} + bool llvm::mustSuppressSpeculation(const LoadInst &LI) { if (!LI.isUnordered()) return true; diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp index efb8d53e8964b..8c1f9c5a3b36f 100644 --- a/llvm/lib/IR/Value.cpp +++ b/llvm/lib/IR/Value.cpp @@ -192,6 +192,10 @@ void Value::dropDroppableUses( } } +void Value::dropDroppableUsesByUser(const User &Usr) { + dropDroppableUses([&](const Use *U) { return U->getUser() == &Usr; }); +} + bool Value::isUsedInBasicBlock(const BasicBlock *BB) const { // This can be computed either by scanning the instructions in BB, or by // scanning the use list of this Value. Both lists can be very long, but diff --git a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp index f079f81a6e8f5..33904e54ac237 100644 --- a/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp +++ b/llvm/lib/Transforms/Utils/PromoteMemoryToRegister.cpp @@ -62,10 +62,6 @@ STATISTIC(NumDeadAlloca, "Number of dead alloca's removed"); STATISTIC(NumPHIInsert, "Number of PHI nodes inserted"); bool llvm::isAllocaPromotable(const AllocaInst *AI) { - // FIXME: If the memory unit is of pointer or integer type, we can permit - // assignments to subsections of the memory unit. - unsigned AS = AI->getType()->getAddressSpace(); - // Only allow direct and non-volatile loads and stores... for (const User *U : AI->users()) { if (const LoadInst *LI = dyn_cast(U)) { @@ -81,19 +77,15 @@ bool llvm::isAllocaPromotable(const AllocaInst *AI) { if (SI->isVolatile()) return false; } else if (const IntrinsicInst *II = dyn_cast(U)) { - if (!II->isLifetimeStartOrEnd()) + if (!II->isLifetimeStartOrEnd() && !II->isDroppable()) return false; } else if (const BitCastInst *BCI = dyn_cast(U)) { - if (BCI->getType() != Type::getInt8PtrTy(U->getContext(), AS)) - return false; - if (!onlyUsedByLifetimeMarkers(BCI)) + if (!onlyUsedByLifetimeMarkersOrDroppableInsts(BCI)) return false; } else if (const GetElementPtrInst *GEPI = dyn_cast(U)) { - if (GEPI->getType() != Type::getInt8PtrTy(U->getContext(), AS)) - return false; if (!GEPI->hasAllZeroIndices()) return false; - if (!onlyUsedByLifetimeMarkers(GEPI)) + if (!onlyUsedByLifetimeMarkersOrDroppableInsts(GEPI)) return false; } else if (const AddrSpaceCastInst *ASCI = dyn_cast(U)) { if (!onlyUsedByLifetimeMarkers(ASCI)) @@ -315,16 +307,38 @@ static void addAssumeNonNull(AssumptionCache *AC, LoadInst *LI) { AC->registerAssumption(CI); } -static void removeLifetimeIntrinsicUsers(AllocaInst *AI) { +static void removeIntrinsicUsers(AllocaInst *AI) { // Knowing that this alloca is promotable, we know that it's safe to kill all // instructions except for load and store. + // Helper to drop the uses of \p I in \p UserI. + auto DropUsesIn = [](Instruction *UserI, Instruction *I, + Instruction::user_iterator &UI, + const Instruction::user_iterator &UE) { + // TODO For now we forget assumed information, this can be improved. + assert(isa(UserI) && + cast(UserI)->getIntrinsicID() == Intrinsic::assume && + "Expected assume"); + + // Skip ahead if User has multiple uses of I. + while (UI != UE && *UI == UserI) + ++UI; + + I->dropDroppableUsesByUser(*UserI); + }; + for (auto UI = AI->user_begin(), UE = AI->user_end(); UI != UE;) { Instruction *I = cast(*UI); ++UI; if (isa(I) || isa(I)) continue; + // Drop the use of AI in droppable instructions. + if (I->isDroppable()) { + DropUsesIn(I, AI, UI, UE); + continue; + } + if (!I->getType()->isVoidTy()) { // The only users of this bitcast/GEP instruction are lifetime intrinsics. // Follow the use/def chain to erase them now instead of leaving it for @@ -332,6 +346,12 @@ static void removeLifetimeIntrinsicUsers(AllocaInst *AI) { for (auto UUI = I->user_begin(), UUE = I->user_end(); UUI != UUE;) { Instruction *Inst = cast(*UUI); ++UUI; + + // Drop the use of I in droppable instructions. + if (Inst->isDroppable()) { + DropUsesIn(Inst, I, UUI, UUE); + continue; + } Inst->eraseFromParent(); } } @@ -547,7 +567,7 @@ void PromoteMem2Reg::run() { assert(AI->getParent()->getParent() == &F && "All allocas should be in the same function, which is same as DF!"); - removeLifetimeIntrinsicUsers(AI); + removeIntrinsicUsers(AI); if (AI->use_empty()) { // If there are no uses of the alloca, just delete it now. diff --git a/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll new file mode 100644 index 0000000000000..ecad226e1d0f0 --- /dev/null +++ b/llvm/test/Transforms/Mem2Reg/ignore-droppable.ll @@ -0,0 +1,85 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -mem2reg -S -o - < %s | FileCheck %s +; RUN: opt -passes=mem2reg -S -o - < %s | FileCheck %s + +declare void @llvm.assume(i1) +declare void @llvm.lifetime.start.p0i8(i64 %size, i8* nocapture %ptr) +declare void @llvm.lifetime.end.p0i8(i64 %size, i8* nocapture %ptr) + +define void @positive_assume_uses(i32* %arg) { +; CHECK-LABEL: @positive_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "nonnull"(i32* [[ARG:%.*]]), "ignore"(i32* undef, i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(i32* undef, i64 8), "nonnull"(i32* [[ARG]]) ] +; CHECK-NEXT: ret void +; + %A = alloca i32 + call void @llvm.assume(i1 true) ["nonnull"(i32* %arg), "align"(i32* %A, i64 2)] + store i32 1, i32* %A + call void @llvm.assume(i1 true) ["align"(i32* %A, i64 8), "nonnull"(i32* %arg)] + ret void +} + +define void @negative_assume_condition_use() { +; CHECK-LABEL: @negative_assume_condition_use( +; CHECK-NEXT: [[A:%.*]] = alloca i32, align 4 +; CHECK-NEXT: [[B:%.*]] = bitcast i32* [[A]] to i8* +; CHECK-NEXT: [[CND:%.*]] = icmp eq i8* [[B]], null +; CHECK-NEXT: call void @llvm.assume(i1 [[CND]]) +; CHECK-NEXT: store i32 1, i32* [[A]], align 4 +; CHECK-NEXT: ret void +; + %A = alloca i32 + %B = bitcast i32* %A to i8* + %cnd = icmp eq i8* %B, null + call void @llvm.assume(i1 %cnd) + store i32 1, i32* %A + ret void +} + +define void @positive_multiple_assume_uses() { +; CHECK-LABEL: @positive_multiple_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"({ i8, i16 }* undef, i64 8), "ignore"({ i8, i16 }* undef, i64 16) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"({ i8, i16 }* undef), "ignore"({ i8, i16 }* undef, i64 2) ] +; CHECK-NEXT: ret void +; + %A = alloca {i8, i16} + call void @llvm.assume(i1 true) ["align"({i8, i16}* %A, i64 8), "align"({i8, i16}* %A, i64 16)] + store {i8, i16} zeroinitializer, {i8, i16}* %A + call void @llvm.assume(i1 true) ["nonnull"({i8, i16}* %A), "align"({i8, i16}* %A, i64 2)] + ret void +} + +define void @positive_gep_assume_uses() { +; CHECK-LABEL: @positive_gep_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(i8* undef, i64 8), "ignore"(i8* undef, i64 16) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 2) ] +; CHECK-NEXT: ret void +; + %A = alloca {i8, i16} + %B = getelementptr {i8, i16}, {i8, i16}* %A, i32 0, i32 0 + call void @llvm.lifetime.start.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["align"(i8* %B, i64 8), "align"(i8* %B, i64 16)] + store {i8, i16} zeroinitializer, {i8, i16}* %A + call void @llvm.lifetime.end.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %B, i64 2)] + ret void +} + +define void @positive_mixed_assume_uses() { +; CHECK-LABEL: @positive_mixed_assume_uses( +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 8), "ignore"(i8* undef, i64 16) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(i8* undef), "ignore"(i8* undef, i64 2), "ignore"(i8* undef) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "ignore"(i32* undef), "ignore"(i32* undef, i64 2), "ignore"(i8* undef) ] +; CHECK-NEXT: ret void +; + %A = alloca i8 + %B = getelementptr i8, i8* %A, i32 0 + %C = bitcast i8* %A to i32* + call void @llvm.lifetime.start.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 8), "align"(i8* %B, i64 16)] + store i8 1, i8* %A + call void @llvm.lifetime.end.p0i8(i64 2, i8* %B) + call void @llvm.assume(i1 true) ["nonnull"(i8* %B), "align"(i8* %A, i64 2), "nonnull"(i8* %A)] + call void @llvm.assume(i1 true) ["nonnull"(i32* %C), "align"(i32* %C, i64 2), "nonnull"(i8* %A)] + ret void +}