diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp index 3b9669179c08a..22d22bcd9d6a8 100644 --- a/llvm/lib/Transforms/Scalar/LICM.cpp +++ b/llvm/lib/Transforms/Scalar/LICM.cpp @@ -928,7 +928,11 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI, canSinkOrHoistInst(I, AA, DT, CurLoop, MSSAU, true, Flags, ORE) && isSafeToExecuteUnconditionally(I, DT, TLI, CurLoop, SafetyInfo, ORE, Preheader->getTerminator(), AC, - AllowSpeculation)) { + AllowSpeculation) && + // May-throw calls can only be hoisted if there are no memory + // writes before them in the loop. Otherwise hoisting would skip + // side effects that must happen before the potential throw. + (!I.mayThrow() || SafetyInfo->doesNotWriteMemoryBefore(I, CurLoop))) { hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo, MSSAU, SE, ORE); HoistedInstructions.push_back(&I); @@ -1209,7 +1213,9 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT, return !Invalidated; } else if (CallInst *CI = dyn_cast(&I)) { // Don't sink calls which can throw. - if (CI->mayThrow()) + // Hoisting if there aren't side effect between the call and the preheader + // This must be checked by the caller + if (Flags.getIsSink() && CI->mayThrow()) return false; // Convergent attribute has been used on operations that involve diff --git a/llvm/test/Transforms/LICM/call-hoisting.ll b/llvm/test/Transforms/LICM/call-hoisting.ll index bb28d1ca93233..1f5ef83d714a4 100644 --- a/llvm/test/Transforms/LICM/call-hoisting.ll +++ b/llvm/test/Transforms/LICM/call-hoisting.ll @@ -472,14 +472,14 @@ declare void @not_nounwind(i32 %v, ptr %p) writeonly argmemonly declare void @not_argmemonly(i32 %v, ptr %p) writeonly nounwind declare void @not_writeonly(i32 %v, ptr %p) argmemonly nounwind -define void @neg_not_nounwind(ptr %loc) { -; CHECK-LABEL: define void @neg_not_nounwind( +define void @hoist_not_nounwind(ptr %loc) { +; CHECK-LABEL: define void @hoist_not_nounwind( ; CHECK-SAME: ptr [[LOC:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: call void @not_nounwind(i32 0, ptr [[LOC]]) ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] -; CHECK-NEXT: call void @not_nounwind(i32 0, ptr [[LOC]]) ; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 ; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200 ; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]] @@ -589,3 +589,115 @@ exit: ret void } +declare i32 @readnone_maythrow(i32) memory(none) + +; A readnone call that may throw but is guaranteed to execute can be hoisted. +define void @hoist_readnone_maythrow(i32 %n, ptr noalias %sink) { +; CHECK-LABEL: define void @hoist_readnone_maythrow( +; CHECK-SAME: i32 [[N:%.*]], ptr noalias [[SINK:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[RET:%.*]] = call i32 @readnone_maythrow(i32 [[N]]) +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: store volatile i32 [[RET]], ptr [[SINK]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200 +; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i32 [0, %entry], [%iv.next, %loop] + %ret = call i32 @readnone_maythrow(i32 %n) + store volatile i32 %ret, ptr %sink + %iv.next = add i32 %iv, 1 + %cmp = icmp slt i32 %iv, 200 + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + +declare i32 @readonly_maythrow(ptr %p) memory(argmem: read) + +; A readonly call that may throw cannot be hoisted if not guaranteed to execute +; (here it's conditionally executed). +define void @no_hoist_readonly_maythrow_conditional(ptr noalias %loc, ptr noalias %sink, i1 %cond) { +; CHECK-LABEL: define void @no_hoist_readonly_maythrow_conditional( +; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[SINK:%.*]], i1 [[COND:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ] +; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[LATCH]] +; CHECK: [[THEN]]: +; CHECK-NEXT: [[RET:%.*]] = call i32 @readonly_maythrow(ptr [[LOC]]) +; CHECK-NEXT: store volatile i32 [[RET]], ptr [[SINK]], align 4 +; CHECK-NEXT: br label %[[LATCH]] +; CHECK: [[LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200 +; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i32 [0, %entry], [%iv.next, %latch] + br i1 %cond, label %then, label %latch + +then: + %ret = call i32 @readonly_maythrow(ptr %loc) + store volatile i32 %ret, ptr %sink + br label %latch + +latch: + %iv.next = add i32 %iv, 1 + %cmp = icmp slt i32 %iv, 200 + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} + +declare void @maythrow_sideeffect() + +; A readonly call that may throw cannot be hoisted if preceded by another +; may-throw instruction in the same block (not guaranteed to execute). +define void @no_hoist_readonly_maythrow_after_icf(ptr noalias %loc, ptr noalias %sink) { +; CHECK-LABEL: define void @no_hoist_readonly_maythrow_after_icf( +; CHECK-SAME: ptr noalias [[LOC:%.*]], ptr noalias [[SINK:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: call void @maythrow_sideeffect() +; CHECK-NEXT: [[RET:%.*]] = call i32 @readonly_maythrow(ptr [[LOC]]) +; CHECK-NEXT: store volatile i32 [[RET]], ptr [[SINK]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[IV]], 200 +; CHECK-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i32 [0, %entry], [%iv.next, %loop] + call void @maythrow_sideeffect() + %ret = call i32 @readonly_maythrow(ptr %loc) + store volatile i32 %ret, ptr %sink + %iv.next = add i32 %iv, 1 + %cmp = icmp slt i32 %iv, 200 + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LICM/hoist-metadata.ll b/llvm/test/Transforms/LICM/hoist-metadata.ll index f25cb966a37fc..d16fcf11bfb34 100644 --- a/llvm/test/Transforms/LICM/hoist-metadata.ll +++ b/llvm/test/Transforms/LICM/hoist-metadata.ll @@ -10,9 +10,9 @@ define void @test_unconditional(i1 %c, ptr dereferenceable(8) align 8 %p) { ; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P]], align 4, !range [[RNG0:![0-9]+]] ; CHECK-NEXT: [[V2:%.*]] = load ptr, ptr [[P]], align 8, !nonnull [[META1:![0-9]+]], !noundef [[META1]] ; CHECK-NEXT: [[V3:%.*]] = load ptr, ptr [[P]], align 8, !dereferenceable [[META2:![0-9]+]], !align [[META2]] +; CHECK-NEXT: call void @foo(i32 [[V1]], ptr [[V2]], ptr [[V3]]) ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: call void @foo(i32 [[V1]], ptr [[V2]], ptr [[V3]]) ; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]] ; CHECK: exit: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/LICM/preheader-safe.ll b/llvm/test/Transforms/LICM/preheader-safe.ll index f97cfbcbf362c..48914829d5f6a 100644 --- a/llvm/test/Transforms/LICM/preheader-safe.ll +++ b/llvm/test/Transforms/LICM/preheader-safe.ll @@ -48,17 +48,18 @@ loop: ; preds = %entry, %for.inc call void @use(i64 %div) br label %loop } - -define void @throw_header_after_rec(ptr %xp, ptr %yp, ptr %cond) { +; The may-throw call (readwrite) cannot be hoisted, but the loads and udiv +; before it are guaranteed to execute and can hoist past it. +define void @throw_header_after_rec(ptr noalias %xp, ptr noalias %yp, ptr %cond) { ; CHECK-LABEL: define void @throw_header_after_rec( -; CHECK-SAME: ptr [[XP:%.*]], ptr [[YP:%.*]], ptr [[COND:%.*]]) { +; CHECK-SAME: ptr noalias [[XP:%.*]], ptr noalias [[YP:%.*]], ptr [[COND:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[X:%.*]] = load i64, ptr [[XP]], align 4 ; CHECK-NEXT: [[Y:%.*]] = load i64, ptr [[YP]], align 4 ; CHECK-NEXT: [[DIV:%.*]] = udiv i64 [[X]], [[Y]] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: -; CHECK-NEXT: call void @use(i64 [[DIV]]) #[[ATTR1:[0-9]+]] +; CHECK-NEXT: call void @use(i64 [[DIV]]) ; CHECK-NEXT: br label %[[LOOP]] ; entry: @@ -68,7 +69,7 @@ loop: ; preds = %entry, %for.inc %x = load i64, ptr %xp %y = load i64, ptr %yp %div = udiv i64 %x, %y - call void @use(i64 %div) readonly + call void @use(i64 %div) br label %loop } @@ -85,7 +86,7 @@ define void @throw_header_after_nonfirst(ptr %xp, ptr %yp, ptr %cond) { ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i64, ptr [[XP]], i64 [[IV]] ; CHECK-NEXT: [[X:%.*]] = load i64, ptr [[GEP]], align 4 ; CHECK-NEXT: [[DIV]] = udiv i64 [[X]], [[Y]] -; CHECK-NEXT: call void @use(i64 [[DIV]]) #[[ATTR1]] +; CHECK-NEXT: call void @use(i64 [[DIV]]) #[[ATTR1:[0-9]+]] ; CHECK-NEXT: br label %[[LOOP]] ; entry: diff --git a/llvm/test/Transforms/LICM/read-only-calls.ll b/llvm/test/Transforms/LICM/read-only-calls.ll index ec8470ab51dfd..a03cf9baf55c5 100644 --- a/llvm/test/Transforms/LICM/read-only-calls.ll +++ b/llvm/test/Transforms/LICM/read-only-calls.ll @@ -48,10 +48,10 @@ define void @test2(ptr %ptr) { ; CHECK-SAME: ptr [[PTR:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR]], align 4 +; CHECK-NEXT: call void @foo(i64 4, ptr [[PTR]]) ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[X_INC:%.*]], %[[LOOP]] ] -; CHECK-NEXT: call void @foo(i64 4, ptr [[PTR]]) ; CHECK-NEXT: [[X_INC]] = add i32 [[X]], [[VAL]] ; CHECK-NEXT: br label %[[LOOP]] ; @@ -66,7 +66,10 @@ loop: br label %loop } -; cannot hoist load since not guaranteed to execute +; cannot hoist load since not guaranteed to execute due to may-throw call before +; it that is not hoistable (readwrite, so it stays in the loop as ICF). +declare void @bar(i64, ptr) + define void @test3(ptr %ptr) { ; CHECK-LABEL: define void @test3( ; CHECK-SAME: ptr [[PTR:%.*]]) { @@ -74,7 +77,7 @@ define void @test3(ptr %ptr) { ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[X:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[X_INC:%.*]], %[[LOOP]] ] -; CHECK-NEXT: call void @foo(i64 4, ptr [[PTR]]) +; CHECK-NEXT: call void @bar(i64 4, ptr [[PTR]]) ; CHECK-NEXT: [[VAL:%.*]] = load i32, ptr [[PTR]], align 4 ; CHECK-NEXT: [[X_INC]] = add i32 [[X]], [[VAL]] ; CHECK-NEXT: br label %[[LOOP]] @@ -84,7 +87,7 @@ entry: loop: %x = phi i32 [ 0, %entry ], [ %x.inc, %loop ] - call void @foo(i64 4, ptr %ptr) + call void @bar(i64 4, ptr %ptr) %val = load i32, ptr %ptr %x.inc = add i32 %x, %val br label %loop