From bff824ed51516bfb023cd147f512a69f70f81f58 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 7 Nov 2025 19:30:46 +0000 Subject: [PATCH 1/3] [InstCombine] Add extra tests. --- .../sink-dereferenceable-assume.ll | 99 ++++++++++++++++++- 1 file changed, 94 insertions(+), 5 deletions(-) diff --git a/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll b/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll index 953132309900b..8751a9fe9c174 100644 --- a/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll +++ b/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll @@ -1,11 +1,11 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 ; RUN: opt -p instcombine -S %s | FileCheck %s -define i64 @test_sink_with_dereferenceable_assume(ptr %p, ptr %q, i1 %cond) { -; CHECK-LABEL: define i64 @test_sink_with_dereferenceable_assume( -; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[COND:%.*]]) { +define i64 @test_dereferenceable_assume(ptr %p, ptr %q, i1 %c.0) { +; CHECK-LABEL: define i64 @test_dereferenceable_assume( +; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] ; CHECK: [[THEN]]: ; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 ; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 @@ -19,7 +19,7 @@ entry: %q_int = ptrtoint ptr %q to i64 %diff = sub i64 %q_int, %p_int call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ] - br i1 %cond, label %then, label %else + br i1 %c.0, label %then, label %else then: ret i64 %diff @@ -28,4 +28,93 @@ else: ret i64 0 } +define i64 @test_sink_with_dereferenceable_assume_same_block_as_user(ptr %p, ptr %q, i1 %c.0) { +; CHECK-LABEL: define i64 @test_sink_with_dereferenceable_assume_same_block_as_user( +; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 +; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ] +; CHECK-NEXT: ret i64 [[DIFF]] +; CHECK: [[ELSE]]: +; CHECK-NEXT: ret i64 0 +; +entry: + %p_int = ptrtoint ptr %p to i64 + %q_int = ptrtoint ptr %q to i64 + %diff = sub i64 %q_int, %p_int + br i1 %c.0, label %then, label %else + +then: + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ] + ret i64 %diff + +else: + ret i64 0 +} + +define i64 @test_sink_with_multiple_users_dominated_by_deref(ptr %p, ptr %q, i1 %c.0, i1 %c.1) { +; CHECK-LABEL: define i64 @test_sink_with_multiple_users_dominated_by_deref( +; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: br i1 [[C_1]], label %[[THEN_2:.*]], label %[[ELSE]] +; CHECK: [[THEN_2]]: +; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 +; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] +; CHECK-NEXT: [[DOUBLED:%.*]] = shl i64 [[DIFF]], 1 +; CHECK-NEXT: ret i64 [[DOUBLED]] +; CHECK: [[ELSE]]: +; CHECK-NEXT: ret i64 0 +; +entry: + %p_int = ptrtoint ptr %p to i64 + %q_int = ptrtoint ptr %q to i64 + %diff = sub i64 %q_int, %p_int + br i1 %c.0, label %then, label %else + +then: + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ] + br i1 %c.1, label %then.2, label %else + +then.2: + %doubled = mul i64 %diff, 2 + ret i64 %doubled + +else: + ret i64 0 +} + +define i64 @test_deref_user_does_not_dominate_other_user(ptr %p, ptr %q, i1 %c.0) { +; CHECK-LABEL: define i64 @test_deref_user_does_not_dominate_other_user( +; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br i1 [[C_0]], label %[[MIDDLE:.*]], label %[[EXIT:.*]] +; CHECK: [[MIDDLE]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 +; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] +; CHECK-NEXT: ret i64 [[DIFF]] +; +entry: + %p_int = ptrtoint ptr %p to i64 + %q_int = ptrtoint ptr %q to i64 + %diff = sub i64 %q_int, %p_int + br i1 %c.0, label %middle, label %exit + +middle: + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %p, i64 %diff) ] + br label %exit + +exit: + ret i64 %diff +} + declare void @llvm.assume(i1 noundef) From 6b524817039d7f076d4389bc4c9136d01abcee7c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 7 Nov 2025 13:35:17 +0000 Subject: [PATCH 2/3] [InstCombine] Don't sink if it would require dropping deref assumptions. Currently sinking assumes in instcombine drops assumes if they would prevent sinking. Removing dereferenceable assumptions earlier on can inhibit vectorization of early-exit loops in practice. Special-case deferenceable assumptions so that they block sinking. This can be combined with a separate change to drop dereferencebale assumptions after vectorization: https://clang.godbolt.org/z/jGqcx3sbs Not sure if there is a better solution. --- .../InstCombine/InstructionCombining.cpp | 12 +++ .../sink-dereferenceable-assume.ll | 4 - .../PhaseOrdering/AArch64/std-find.ll | 102 ++++++++---------- 3 files changed, 58 insertions(+), 60 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index b158e0f626850..cea5be2468feb 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -5413,6 +5413,18 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I, return false; } + // Do not sink if there are dereferenceable assumes that would be removed. + for (User *User : I->users()) { + auto *CI = dyn_cast(User); + if (!CI || CI->getParent() == DestBlock) + continue; + + if (CI->getIntrinsicID() == Intrinsic::assume && + CI->getOperandBundle("dereferenceable")) { + return false; + } + } + I->dropDroppableUses([&](const Use *U) { auto *I = dyn_cast(U->getUser()); if (I && I->getParent() != DestBlock) { diff --git a/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll b/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll index 8751a9fe9c174..8b9779f3df63a 100644 --- a/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll +++ b/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll @@ -5,11 +5,7 @@ define i64 @test_dereferenceable_assume(ptr %p, ptr %q, i1 %c.0) { ; CHECK-LABEL: define i64 @test_dereferenceable_assume( ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] -; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] ; CHECK: [[THEN]]: -; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 -; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 -; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] ; CHECK-NEXT: ret i64 [[DIFF]] ; CHECK: [[ELSE]]: ; CHECK-NEXT: ret i64 0 diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll index 33e3e83770e7f..e9149795954ec 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll @@ -133,75 +133,65 @@ define ptr @std_find_caller(ptr noundef %first, ptr noundef %last) { ; CHECK-LABEL: define noundef ptr @std_find_caller( ; CHECK-SAME: ptr noundef [[FIRST:%.*]], ptr noundef [[LAST:%.*]]) local_unnamed_addr #[[ATTR0]] { ; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[FIRST3:%.*]] = ptrtoint ptr [[FIRST]] to i64 +; CHECK-NEXT: [[LAST_I64:%.*]] = ptrtoint ptr [[LAST]] to i64 +; CHECK-NEXT: [[PTR_SUB:%.*]] = sub i64 [[LAST_I64]], [[FIRST3]] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[FIRST]], i64 2) ] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[LAST]], i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[FIRST]], i64 [[PTR_SUB]]) ] ; CHECK-NEXT: [[PRE_I:%.*]] = icmp eq ptr [[FIRST]], [[LAST]] ; CHECK-NEXT: br i1 [[PRE_I]], label %[[STD_FIND_GENERIC_IMPL_EXIT:.*]], label %[[LOOP_HEADER_I_PREHEADER:.*]] ; CHECK: [[LOOP_HEADER_I_PREHEADER]]: -; CHECK-NEXT: [[LAST2:%.*]] = ptrtoint ptr [[LAST]] to i64 -; CHECK-NEXT: [[FIRST3:%.*]] = ptrtoint ptr [[FIRST]] to i64 -; CHECK-NEXT: [[LAST_I64:%.*]] = ptrtoint ptr [[LAST]] to i64 -; CHECK-NEXT: [[FIRST1:%.*]] = ptrtoint ptr [[FIRST]] to i64 -; CHECK-NEXT: [[PTR_SUB:%.*]] = sub i64 [[LAST_I64]], [[FIRST1]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[PTR_SUB]] -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[LAST2]], -2 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[LAST_I64]], -2 ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[FIRST3]] ; CHECK-NEXT: [[TMP2:%.*]] = lshr exact i64 [[TMP1]], 1 ; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 [[TMP2]], 1 -; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP3]], 3 -; CHECK-NEXT: [[TMP4:%.*]] = and i64 [[TMP1]], 6 -; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[TMP4]], 6 -; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label %[[LOOP_HEADER_I_PROL_LOOPEXIT:.*]], label %[[LOOP_HEADER_I_PROL:.*]] -; CHECK: [[LOOP_HEADER_I_PROL]]: -; CHECK-NEXT: [[PTR_IV_I_PROL:%.*]] = phi ptr [ [[PTR_IV_NEXT_I_PROL:%.*]], %[[LOOP_LATCH_I_PROL:.*]] ], [ [[FIRST]], %[[LOOP_HEADER_I_PREHEADER]] ] -; CHECK-NEXT: [[PROL_ITER:%.*]] = phi i64 [ [[PROL_ITER_NEXT:%.*]], %[[LOOP_LATCH_I_PROL]] ], [ 0, %[[LOOP_HEADER_I_PREHEADER]] ] -; CHECK-NEXT: [[L_I_PROL:%.*]] = load i16, ptr [[PTR_IV_I_PROL]], align 2 -; CHECK-NEXT: [[C_1_I_PROL:%.*]] = icmp eq i16 [[L_I_PROL]], 1 -; CHECK-NEXT: br i1 [[C_1_I_PROL]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_LATCH_I_PROL]] -; CHECK: [[LOOP_LATCH_I_PROL]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_PROL]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I_PROL]], i64 2 -; CHECK-NEXT: [[PROL_ITER_NEXT]] = add i64 [[PROL_ITER]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP1]], 158 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[LOOP_HEADER_I_PREHEADER2:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[TMP3]], -8 +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[PROL_ITER_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2 +; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <8 x i16> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD_FR]], splat (i16 1) +; CHECK-NEXT: [[PROL_ITER_NEXT]] = add nuw i64 [[INDEX]], 8 +; CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i1> [[TMP4]] to i8 +; CHECK-NEXT: [[TMP6:%.*]] = icmp ne i8 [[TMP5]], 0 ; CHECK-NEXT: [[PROL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[PROL_ITER_NEXT]], [[XTRAITER]] -; CHECK-NEXT: br i1 [[PROL_ITER_CMP_NOT]], label %[[LOOP_HEADER_I_PROL_LOOPEXIT]], label %[[LOOP_HEADER_I_PROL]], !llvm.loop [[LOOP3:![0-9]+]] -; CHECK: [[LOOP_HEADER_I_PROL_LOOPEXIT]]: -; CHECK-NEXT: [[PTR_IV_I_UNR:%.*]] = phi ptr [ [[FIRST]], %[[LOOP_HEADER_I_PREHEADER]] ], [ [[PTR_IV_NEXT_I_PROL]], %[[LOOP_LATCH_I_PROL]] ] -; CHECK-NEXT: [[TMP5:%.*]] = icmp ult i64 [[TMP1]], 6 -; CHECK-NEXT: br i1 [[TMP5]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_HEADER_I:.*]] +; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[PROL_ITER_CMP_NOT]] +; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[MIDDLE_SPLIT]]: +; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[XTRAITER]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[TMP9]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[XTRAITER]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_HEADER_I_PREHEADER2]] +; CHECK: [[LOOP_HEADER_I_PREHEADER2]]: +; CHECK-NEXT: [[PTR_IV_I_PH:%.*]] = phi ptr [ [[FIRST]], %[[LOOP_HEADER_I_PREHEADER]] ], [ [[TMP10]], %[[MIDDLE_BLOCK]] ] +; CHECK-NEXT: br label %[[LOOP_HEADER_I:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: [[TMP11:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP4]], i1 true) +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 1 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[TMP13]] +; CHECK-NEXT: br label %[[STD_FIND_GENERIC_IMPL_EXIT]] ; CHECK: [[LOOP_HEADER_I]]: -; CHECK-NEXT: [[PTR_IV_I:%.*]] = phi ptr [ [[PTR_IV_NEXT_I_3:%.*]], %[[LOOP_LATCH_I_3:.*]] ], [ [[PTR_IV_I_UNR]], %[[LOOP_HEADER_I_PROL_LOOPEXIT]] ] +; CHECK-NEXT: [[PTR_IV_I:%.*]] = phi ptr [ [[PTR_IV_NEXT_I:%.*]], %[[LOOP_LATCH_I:.*]] ], [ [[PTR_IV_I_PH]], %[[LOOP_HEADER_I_PREHEADER2]] ] ; CHECK-NEXT: [[L_I:%.*]] = load i16, ptr [[PTR_IV_I]], align 2 ; CHECK-NEXT: [[C_1_I:%.*]] = icmp eq i16 [[L_I]], 1 -; CHECK-NEXT: br i1 [[C_1_I]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_LATCH_I:.*]] +; CHECK-NEXT: br i1 [[C_1_I]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_LATCH_I]] ; CHECK: [[LOOP_LATCH_I]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 2 -; CHECK-NEXT: [[L_I_1:%.*]] = load i16, ptr [[PTR_IV_NEXT_I]], align 2 -; CHECK-NEXT: [[C_1_I_1:%.*]] = icmp eq i16 [[L_I_1]], 1 -; CHECK-NEXT: br i1 [[C_1_I_1]], label %[[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT11:.*]], label %[[LOOP_LATCH_I_1:.*]] -; CHECK: [[LOOP_LATCH_I_1]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_1:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 4 -; CHECK-NEXT: [[L_I_2:%.*]] = load i16, ptr [[PTR_IV_NEXT_I_1]], align 2 -; CHECK-NEXT: [[C_1_I_2:%.*]] = icmp eq i16 [[L_I_2]], 1 -; CHECK-NEXT: br i1 [[C_1_I_2]], label %[[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT9:.*]], label %[[LOOP_LATCH_I_2:.*]] -; CHECK: [[LOOP_LATCH_I_2]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_2:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 6 -; CHECK-NEXT: [[L_I_3:%.*]] = load i16, ptr [[PTR_IV_NEXT_I_2]], align 2 -; CHECK-NEXT: [[C_1_I_3:%.*]] = icmp eq i16 [[L_I_3]], 1 -; CHECK-NEXT: br i1 [[C_1_I_3]], label %[[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT7:.*]], label %[[LOOP_LATCH_I_3]] -; CHECK: [[LOOP_LATCH_I_3]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_3]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 8 -; CHECK-NEXT: [[EC_I_3:%.*]] = icmp eq ptr [[PTR_IV_NEXT_I_3]], [[LAST]] -; CHECK-NEXT: br i1 [[EC_I_3]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_HEADER_I]] -; CHECK: [[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT7]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_2_LE:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 6 -; CHECK-NEXT: br label %[[STD_FIND_GENERIC_IMPL_EXIT]] -; CHECK: [[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT9]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_1_LE:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 4 -; CHECK-NEXT: br label %[[STD_FIND_GENERIC_IMPL_EXIT]] -; CHECK: [[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT11]]: -; CHECK-NEXT: [[PTR_IV_NEXT_I_LE:%.*]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 2 -; CHECK-NEXT: br label %[[STD_FIND_GENERIC_IMPL_EXIT]] +; CHECK-NEXT: [[PTR_IV_NEXT_I]] = getelementptr inbounds nuw i8, ptr [[PTR_IV_I]], i64 2 +; CHECK-NEXT: [[EC_I:%.*]] = icmp eq ptr [[PTR_IV_NEXT_I]], [[LAST]] +; CHECK-NEXT: br i1 [[EC_I]], label %[[STD_FIND_GENERIC_IMPL_EXIT]], label %[[LOOP_HEADER_I]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: [[STD_FIND_GENERIC_IMPL_EXIT]]: -; CHECK-NEXT: [[RES_I:%.*]] = phi ptr [ [[FIRST]], %[[ENTRY]] ], [ [[SCEVGEP]], %[[LOOP_HEADER_I_PROL_LOOPEXIT]] ], [ [[PTR_IV_NEXT_I_2_LE]], %[[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT7]] ], [ [[PTR_IV_NEXT_I_1_LE]], %[[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT9]] ], [ [[PTR_IV_NEXT_I_LE]], %[[STD_FIND_GENERIC_IMPL_EXIT_LOOPEXIT_UNR_LCSSA_LOOPEXIT_SPLIT_LOOP_EXIT11]] ], [ [[SCEVGEP]], %[[LOOP_LATCH_I_3]] ], [ [[PTR_IV_I]], %[[LOOP_HEADER_I]] ], [ [[PTR_IV_I_PROL]], %[[LOOP_HEADER_I_PROL]] ] +; CHECK-NEXT: [[RES_I:%.*]] = phi ptr [ [[FIRST]], %[[ENTRY]] ], [ [[SCEVGEP]], %[[MIDDLE_BLOCK]] ], [ [[TMP14]], %[[VECTOR_EARLY_EXIT]] ], [ [[SCEVGEP]], %[[LOOP_LATCH_I]] ], [ [[PTR_IV_I]], %[[LOOP_HEADER_I]] ] ; CHECK-NEXT: ret ptr [[RES_I]] ; entry: @@ -241,6 +231,6 @@ declare void @llvm.assume(i1 noundef) ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} ; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} -; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META4:![0-9]+]]} -; CHECK: [[META4]] = !{!"llvm.loop.unroll.disable"} +; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]} +; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META2]], [[META1]]} ;. From ae29bfe2b1a55b4c9058f8038ce278db0c3be009 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 7 Nov 2025 19:27:34 +0000 Subject: [PATCH 3/3] !fixup move code to getOptionalSinkBlockForInst. --- .../InstCombine/InstructionCombining.cpp | 23 ++++++++----------- .../sink-dereferenceable-assume.ll | 19 ++++++++++----- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index cea5be2468feb..fbefd4ab84d99 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -5413,18 +5413,6 @@ bool InstCombinerImpl::tryToSinkInstruction(Instruction *I, return false; } - // Do not sink if there are dereferenceable assumes that would be removed. - for (User *User : I->users()) { - auto *CI = dyn_cast(User); - if (!CI || CI->getParent() == DestBlock) - continue; - - if (CI->getIntrinsicID() == Intrinsic::assume && - CI->getOperandBundle("dereferenceable")) { - return false; - } - } - I->dropDroppableUses([&](const Use *U) { auto *I = dyn_cast(U->getUser()); if (I && I->getParent() != DestBlock) { @@ -5636,8 +5624,15 @@ bool InstCombinerImpl::run() { for (Use &U : I->uses()) { User *User = U.getUser(); - if (User->isDroppable()) - continue; + if (User->isDroppable()) { + // Do not sink if there are dereferenceable assumes that would be + // removed. + auto II = dyn_cast(User); + if (II->getIntrinsicID() != Intrinsic::assume || + !II->getOperandBundle("dereferenceable")) + continue; + } + if (NumUsers > MaxSinkNumUsers) return std::nullopt; diff --git a/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll b/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll index 8b9779f3df63a..8ceb310fc8f71 100644 --- a/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll +++ b/llvm/test/Transforms/InstCombine/sink-dereferenceable-assume.ll @@ -5,6 +5,11 @@ define i64 @test_dereferenceable_assume(ptr %p, ptr %q, i1 %c.0) { ; CHECK-LABEL: define i64 @test_dereferenceable_assume( ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 +; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ] +; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] ; CHECK: [[THEN]]: ; CHECK-NEXT: ret i64 [[DIFF]] ; CHECK: [[ELSE]]: @@ -56,13 +61,14 @@ define i64 @test_sink_with_multiple_users_dominated_by_deref(ptr %p, ptr %q, i1 ; CHECK-LABEL: define i64 @test_sink_with_multiple_users_dominated_by_deref( ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]], i1 [[C_1:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 +; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] ; CHECK-NEXT: br i1 [[C_0]], label %[[THEN:.*]], label %[[ELSE:.*]] ; CHECK: [[THEN]]: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ] ; CHECK-NEXT: br i1 [[C_1]], label %[[THEN_2:.*]], label %[[ELSE]] ; CHECK: [[THEN_2]]: -; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 -; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 -; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] ; CHECK-NEXT: [[DOUBLED:%.*]] = shl i64 [[DIFF]], 1 ; CHECK-NEXT: ret i64 [[DOUBLED]] ; CHECK: [[ELSE]]: @@ -90,13 +96,14 @@ define i64 @test_deref_user_does_not_dominate_other_user(ptr %p, ptr %q, i1 %c.0 ; CHECK-LABEL: define i64 @test_deref_user_does_not_dominate_other_user( ; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], i1 [[C_0:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 +; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] ; CHECK-NEXT: br i1 [[C_0]], label %[[MIDDLE:.*]], label %[[EXIT:.*]] ; CHECK: [[MIDDLE]]: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[P]], i64 [[DIFF]]) ] ; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: -; CHECK-NEXT: [[Q_INT:%.*]] = ptrtoint ptr [[Q]] to i64 -; CHECK-NEXT: [[P_INT:%.*]] = ptrtoint ptr [[P]] to i64 -; CHECK-NEXT: [[DIFF:%.*]] = sub i64 [[Q_INT]], [[P_INT]] ; CHECK-NEXT: ret i64 [[DIFF]] ; entry: