From f433ff80e48c523951f8e0dcb8b49f60a2a4087b Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 19 Aug 2025 13:22:45 +0100 Subject: [PATCH 01/60] [LoopIdiom] Add test for simplifying SCEV during expansion with flags. (cherry picked from commit 1217c8226b09657800bb8711e2be49a143de9dca) --- .../expand-scev-expand-simplifications.ll | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll diff --git a/llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll b/llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll new file mode 100644 index 0000000000000..9a59e5a8ccabb --- /dev/null +++ b/llvm/test/Transforms/LoopIdiom/expand-scev-expand-simplifications.ll @@ -0,0 +1,42 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -p loop-idiom -S %s | FileCheck %s + +target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-n32:64-S128-Fn32" + +define void @test_simplify_scev_during_expansion_flags(i64 %start) { +; CHECK-LABEL: define void @test_simplify_scev_during_expansion_flags( +; CHECK-SAME: i64 [[START:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[START_NEG:%.*]] = sub i64 0, [[START]] +; CHECK-NEXT: [[START_MUL:%.*]] = ashr exact i64 [[START_NEG]], 2 +; CHECK-NEXT: [[TMP0:%.*]] = shl nsw i64 [[START_MUL]], 2 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 404, [[TMP0]] +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 4 null, i8 0, i64 [[TMP1]], i1 false) +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[START_MUL]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ null, %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 4 +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 100 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + %start.neg = sub i64 0, %start + %start.mul = ashr exact i64 %start.neg, 2 + br label %loop + +loop: + %iv = phi i64 [ %start.mul, %entry ], [ %iv.next, %loop ] + %ptr.iv = phi ptr [ null, %entry ], [ %ptr.iv.next, %loop ] + store i32 0, ptr %ptr.iv, align 4 + %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 4 + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, 100 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} From b73cf051e712353fe459ede8401f32c219b04b55 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 25 Aug 2025 22:09:15 +0100 Subject: [PATCH 02/60] [IndVars,LV] Add tests for missed SCEV simplifications with muls. (cherry picked from commit f0df62f7b65d2da0813c4e7ced813250da3b9eef) --- .../rewrite-loop-exit-values-phi.ll | 41 +++++++ .../Transforms/LoopVectorize/runtime-check.ll | 101 ++++++++++++++++++ 2 files changed, 142 insertions(+) diff --git a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll index b1043b48d40ac..e1a042747e0ac 100644 --- a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll +++ b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll @@ -68,3 +68,44 @@ latch: ; preds = %inner_exit end: ; preds = %header ret void } + + +declare void @foo() + +define i64 @narow_canonical_iv_wide_multiplied_iv(i32 %x, i64 %y, ptr %0) { +; CHECK-LABEL: @narow_canonical_iv_wide_multiplied_iv( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[SMAX:%.*]] = tail call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 1) +; CHECK-NEXT: [[MUL_Y:%.*]] = shl i64 [[Y:%.*]], 1 +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[IV_MUL:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[IV_MUL_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[IV_MUL_NEXT]] = add i64 [[IV_MUL]], [[MUL_Y]] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[SMAX]] +; CHECK-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT:%.*]] +; CHECK: exit: +; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ [[IV_MUL_NEXT]], [[LOOP]] ] +; CHECK-NEXT: ret i64 [[TMP6]] +; +entry: + %smax = tail call i32 @llvm.smax.i32(i32 %x, i32 1) + %mul.y = shl i64 %y, 1 + br label %loop + +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] + %iv.mul = phi i64 [ 1, %entry ], [ %iv.mul.next, %loop ] + %iv.mul.next = add i64 %iv.mul, %mul.y + call void @foo() + %iv.next = add i32 %iv, 1 + %ec = icmp ult i32 %iv.next, %smax + br i1 %ec, label %loop, label %exit + +exit: + ret i64 %iv.mul.next +} + +declare i32 @llvm.smax.i32(i32, i32) diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll index 22d9a5363bee6..c05750ee83d92 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll @@ -475,6 +475,107 @@ for.body: br i1 %exitcond, label %for.cond.cleanup, label %for.body, !llvm.loop !12 } +declare i1 @cond() + +define void @test_scev_check_mul_add_expansion(ptr %out, ptr %in, i32 %len, i32 %d) { +; CHECK-LABEL: @test_scev_check_mul_add_expansion( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[PRE_1:%.*]] = icmp samesign ugt i32 [[D:%.*]], 5 +; CHECK-NEXT: tail call void @llvm.assume(i1 [[PRE_1]]) +; CHECK-NEXT: [[PRE_2:%.*]] = icmp ult i32 [[D]], 7 +; CHECK-NEXT: tail call void @llvm.assume(i1 [[PRE_2]]) +; CHECK-NEXT: [[PRE_3:%.*]] = icmp slt i32 [[D]], [[LEN:%.*]] +; CHECK-NEXT: tail call void @llvm.assume(i1 [[PRE_3]]) +; CHECK-NEXT: [[SMAX3:%.*]] = call i32 @llvm.smax.i32(i32 [[LEN]], i32 7) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[SMAX3]], -6 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp slt i32 [[LEN]], 10 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] +; CHECK: vector.memcheck: +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT:%.*]], i64 12 +; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[LEN]], -7 +; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[TMP1]] to i64 +; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[OUT]], i64 [[TMP3]] +; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP4]], i64 14 +; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[IN:%.*]], i64 4 +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP1]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i32 [[TMP0]], -4 +; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[N_VEC]], 6 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[INDEX]], 6 +; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[OFFSET_IDX]] to i64 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[OUT]], i64 [[TMP6]] +; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP7]], align 2, !alias.scope [[META42:![0-9]+]], !noalias [[META45:![0-9]+]] +; CHECK-NEXT: store i32 0, ptr [[IN]], align 4, !alias.scope [[META45]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ 6, [[ENTRY:%.*]] ], [ 6, [[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[TMP9:%.*]] = sext i32 [[IV]] to i64 +; CHECK-NEXT: [[ARRAYIDX80:%.*]] = getelementptr i16, ptr [[OUT]], i64 [[TMP9]] +; CHECK-NEXT: store i16 0, ptr [[ARRAYIDX80]], align 2 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: store i32 0, ptr [[IN]], align 4 +; CHECK-NEXT: [[CMP7_NOT:%.*]] = icmp sgt i32 [[LEN]], [[IV_NEXT]] +; CHECK-NEXT: br i1 [[CMP7_NOT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP48:![0-9]+]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +; FORCED_OPTSIZE-LABEL: @test_scev_check_mul_add_expansion( +; FORCED_OPTSIZE-NEXT: entry: +; FORCED_OPTSIZE-NEXT: [[PRE_1:%.*]] = icmp sgt i32 [[D:%.*]], 5 +; FORCED_OPTSIZE-NEXT: tail call void @llvm.assume(i1 [[PRE_1]]) +; FORCED_OPTSIZE-NEXT: [[PRE_2:%.*]] = icmp samesign ule i32 [[D]], 6 +; FORCED_OPTSIZE-NEXT: tail call void @llvm.assume(i1 [[PRE_2]]) +; FORCED_OPTSIZE-NEXT: [[PRE_3:%.*]] = icmp slt i32 [[D]], [[LEN:%.*]] +; FORCED_OPTSIZE-NEXT: tail call void @llvm.assume(i1 [[PRE_3]]) +; FORCED_OPTSIZE-NEXT: br label [[LOOP:%.*]] +; FORCED_OPTSIZE: loop: +; FORCED_OPTSIZE-NEXT: [[IV:%.*]] = phi i32 [ 6, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] +; FORCED_OPTSIZE-NEXT: [[ARRAYIDX80:%.*]] = getelementptr i16, ptr [[OUT:%.*]], i32 [[IV]] +; FORCED_OPTSIZE-NEXT: store i16 0, ptr [[ARRAYIDX80]], align 2 +; FORCED_OPTSIZE-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; FORCED_OPTSIZE-NEXT: store i32 0, ptr [[IN:%.*]], align 4 +; FORCED_OPTSIZE-NEXT: [[CMP7_NOT:%.*]] = icmp sgt i32 [[LEN]], [[IV_NEXT]] +; FORCED_OPTSIZE-NEXT: br i1 [[CMP7_NOT]], label [[LOOP]], label [[EXIT:%.*]] +; FORCED_OPTSIZE: exit: +; FORCED_OPTSIZE-NEXT: ret void +; +entry: + %pre.1 = icmp sgt i32 %d, 5 + tail call void @llvm.assume(i1 %pre.1) + %pre.2 = icmp samesign ule i32 %d, 6 + tail call void @llvm.assume(i1 %pre.2) + %pre.3 = icmp slt i32 %d, %len + tail call void @llvm.assume(i1 %pre.3) + br label %loop + +loop: + %iv = phi i32 [ 6, %entry ], [ %iv.next, %loop ] + %arrayidx80 = getelementptr i16, ptr %out, i32 %iv + store i16 0, ptr %arrayidx80, align 2 + %iv.next = add nuw nsw i32 %iv, 1 + store i32 0, ptr %in, align 4 + %cmp7.not = icmp sgt i32 %len, %iv.next + br i1 %cmp7.not, label %loop, label %exit + +exit: + ret void +} + ; CHECK: !9 = !DILocation(line: 101, column: 1, scope: !{{.*}}) !llvm.module.flags = !{!0, !1} From efc6ff0da17043457f0909dedbfc257bd0c9731f Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 17 Jul 2025 16:47:54 +0200 Subject: [PATCH 03/60] [SCEV] Try to re-use existing LCSSA phis when expanding SCEVAddRecExpr. (#147214) If an AddRec is expanded outside a loop with a single exit block, check if any of the (lcssa) phi nodes in the exit block match the AddRec. If that's the case, simply use the existing lcssa phi. This can reduce the number of instruction created for SCEV expansions, mainly for runtime checks generated by the loop vectorizer. Compile-time impact should be mostly neutral https://llvm-compile-time-tracker.com/compare.php?from=48c7a3187f9831304a38df9bdb3b4d5bf6b6b1a2&to=cf9d039a7b0db5d0d912e0e2c01b19c2a653273a&stat=instructions:u PR: https://github.com/llvm/llvm-project/pull/147214 (cherry-picked from 46357438baefbdcf630abc5d74565afcbf1c48dd) --- .../Utils/ScalarEvolutionExpander.h | 1 + .../Utils/ScalarEvolutionExpander.cpp | 23 +++++++++++++++++++ .../reuse-lcssa-phi-scev-expansion.ll | 4 +--- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h index a101151eed7cc..39fef921a9590 100644 --- a/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h +++ b/llvm/include/llvm/Transforms/Utils/ScalarEvolutionExpander.h @@ -530,6 +530,7 @@ class SCEVExpander : public SCEVVisitor { bool isExpandedAddRecExprPHI(PHINode *PN, Instruction *IncV, const Loop *L); + Value *tryToReuseLCSSAPhi(const SCEVAddRecExpr *S); Value *expandAddRecExprLiterally(const SCEVAddRecExpr *); PHINode *getAddRecExprPHILiterally(const SCEVAddRecExpr *Normalized, const Loop *L, Type *&TruncTy, diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index bf457194bfd8e..ace74d086e8db 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1223,6 +1223,24 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) { return Result; } +Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) { + const Loop *L = S->getLoop(); + BasicBlock *EB = L->getExitBlock(); + if (!EB || !EB->getSinglePredecessor() || + !SE.DT.dominates(EB, Builder.GetInsertBlock())) + return nullptr; + + for (auto &PN : EB->phis()) { + if (!SE.isSCEVable(PN.getType()) || PN.getType() != S->getType()) + continue; + auto *ExitV = SE.getSCEV(&PN); + if (S == ExitV) + return &PN; + } + + return nullptr; +} + Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) { // In canonical mode we compute the addrec as an expression of a canonical IV // using evaluateAtIteration and expand the resulting SCEV expression. This @@ -1262,6 +1280,11 @@ Value *SCEVExpander::visitAddRecExpr(const SCEVAddRecExpr *S) { return V; } + // If S is expanded outside the defining loop, check if there is a + // matching LCSSA phi node for it. + if (Value *V = tryToReuseLCSSAPhi(S)) + return V; + // {X,+,F} --> X + {0,+,F} if (!S->getStart()->isZero()) { if (isa(S->getType())) { diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll index 2747895f06a7b..ce4270dc4b7fa 100644 --- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll @@ -18,11 +18,9 @@ define void @reuse_lcssa_phi_for_add_rec1(ptr %head) { ; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 ; CHECK-NEXT: br i1 [[EC_1]], label %[[PH:.*]], label %[[LOOP_1]] ; CHECK: [[PH]]: -; CHECK-NEXT: [[IV_2_LCSSA:%.*]] = phi i32 [ [[IV_2]], %[[LOOP_1]] ] ; CHECK-NEXT: [[IV_LCSSA:%.*]] = phi i64 [ [[IV]], %[[LOOP_1]] ] -; CHECK-NEXT: [[IV_2_NEXT_LCSSA:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_1]] ] +; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ [[IV_2_NEXT]], %[[LOOP_1]] ] ; CHECK-NEXT: [[SRC_2:%.*]] = tail call noalias noundef dereferenceable_or_null(8) ptr @calloc(i64 1, i64 8) -; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[IV_2_LCSSA]], 1 ; CHECK-NEXT: [[SMIN:%.*]] = call i32 @llvm.smin.i32(i32 [[TMP0]], i32 1) ; CHECK-NEXT: [[TMP1:%.*]] = sub i32 [[TMP0]], [[SMIN]] ; CHECK-NEXT: [[TMP2:%.*]] = zext i32 [[TMP1]] to i64 From 6f4e97f263cc3e12a7d9056ed15df9930216e186 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 21 Jul 2025 11:40:55 +0100 Subject: [PATCH 04/60] [LAA] Add test variant with backward dep with overlap in loop. The original test @backward_dep_known_distance_less_than_btc was incorrectly named, as all loads are completely before the first store. Add a variant where this is not the case: @backward_dep_known_distance_less_than_btc (cherry picked from commit a11c5dd34b2cfa975934250b13628cac919fb7cb) --- ...es-safe-dep-due-to-backedge-taken-count.ll | 43 ++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll b/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll index 0d1b0829c09da..72b620aad31dc 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll @@ -106,8 +106,10 @@ exit: ret void } -define void @backward_dep_known_distance_less_than_btc(ptr %A) { -; CHECK-LABEL: 'backward_dep_known_distance_less_than_btc' +; TOOD: The loop should be safe without dependence, as all accesses to %l are +; completely before the first store. +define void @backward_dep_known_safe_due_to_backedge_taken_count(ptr %A) { +; CHECK-LABEL: 'backward_dep_known_safe_due_to_backedge_taken_count' ; CHECK-NEXT: loop: ; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 8160 bits ; CHECK-NEXT: Dependences: @@ -142,3 +144,40 @@ loop: exit: ret void } + +define void @backward_dep_known_distance_less_than_btc(ptr %A) { +; CHECK-LABEL: 'backward_dep_known_distance_less_than_btc' +; CHECK-NEXT: loop: +; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 4064 bits +; CHECK-NEXT: Dependences: +; CHECK-NEXT: BackwardVectorizable: +; CHECK-NEXT: %l = load i32, ptr %gep, align 4 -> +; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + %A.510 = getelementptr inbounds i32, ptr %A, i64 510 + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %iv.mul.2 = shl nuw nsw i64 %iv, 1 + %gep = getelementptr inbounds i32, ptr %A, i64 %iv.mul.2 + %l = load i32, ptr %gep, align 4 + %add = add nsw i32 %l, 5 + %gep.mul.2 = getelementptr inbounds i32, ptr %A.510, i64 %iv + store i32 %add, ptr %gep.mul.2, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %exitcond.not = icmp eq i64 %iv.next, 256 + br i1 %exitcond.not, label %exit, label %loop + +exit: + ret void +} From 86aa20875ea2181d922090e81ab227310e5a433f Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 23 Jul 2025 10:33:34 +0200 Subject: [PATCH 05/60] [SCEV] Don't require NUW at first add when checking A+C1 < (A+C2) (#149795) Relax the NUW requirements for isKnownPredicateViaNoOverflow, if the second operand (Y) is an ADD. The code only simplifies the condition if C1 < C2, so if the second ADD is NUW, it doesn't matter whether the first operand also has the NUW flag, as it cannot wrap if C1 < C2. https://alive2.llvm.org/ce/z/b3dM7N PR: https://github.com/llvm/llvm-project/pull/149795 (cherry picked from commit 6c50e2b2dda185816b3a4d65cef6771dad5113d8) --- llvm/lib/Analysis/ScalarEvolution.cpp | 21 ++++++++++++------- ...es-safe-dep-due-to-backedge-taken-count.ll | 8 +------ ...endence-distance-different-access-sizes.ll | 13 +----------- 3 files changed, 16 insertions(+), 26 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 221468a2d1a84..1b6ebdd4eb111 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -11418,8 +11418,7 @@ bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred, XNonConstOp = X; XFlagsPresent = ExpectedFlags; } - if (!isa(XConstOp) || - (XFlagsPresent & ExpectedFlags) != ExpectedFlags) + if (!isa(XConstOp)) return false; if (!splitBinaryAdd(Y, YConstOp, YNonConstOp, YFlagsPresent)) { @@ -11428,12 +11427,20 @@ bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred, YFlagsPresent = ExpectedFlags; } - if (!isa(YConstOp) || - (YFlagsPresent & ExpectedFlags) != ExpectedFlags) + if (YNonConstOp != XNonConstOp) return false; - if (YNonConstOp != XNonConstOp) + if (!isa(YConstOp)) + return false; + + // When matching ADDs with NUW flags (and unsigned predicates), only the + // second ADD (with the larger constant) requires NUW. + if ((YFlagsPresent & ExpectedFlags) != ExpectedFlags) + return false; + if (ExpectedFlags != SCEV::FlagNUW && + (XFlagsPresent & ExpectedFlags) != ExpectedFlags) { return false; + } OutC1 = cast(XConstOp)->getAPInt(); OutC2 = cast(YConstOp)->getAPInt(); @@ -11472,7 +11479,7 @@ bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred, std::swap(LHS, RHS); [[fallthrough]]; case ICmpInst::ICMP_ULE: - // (X + C1) u<= (X + C2) for C1 u<= C2. + // (X + C1) u<= (X + C2) for C1 u<= C2. if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ule(C2)) return true; @@ -11482,7 +11489,7 @@ bool ScalarEvolution::isKnownPredicateViaNoOverflow(CmpPredicate Pred, std::swap(LHS, RHS); [[fallthrough]]; case ICmpInst::ICMP_ULT: - // (X + C1) u< (X + C2) if C1 u< C2. + // (X + C1) u< (X + C2) if C1 u< C2. if (MatchBinaryAddToConst(LHS, RHS, C1, C2, SCEV::FlagNUW) && C1.ult(C2)) return true; break; diff --git a/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll b/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll index 72b620aad31dc..311de84993001 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/different-strides-safe-dep-due-to-backedge-taken-count.ll @@ -106,17 +106,11 @@ exit: ret void } -; TOOD: The loop should be safe without dependence, as all accesses to %l are -; completely before the first store. define void @backward_dep_known_safe_due_to_backedge_taken_count(ptr %A) { ; CHECK-LABEL: 'backward_dep_known_safe_due_to_backedge_taken_count' ; CHECK-NEXT: loop: -; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 8160 bits +; CHECK-NEXT: Memory dependences are safe ; CHECK-NEXT: Dependences: -; CHECK-NEXT: BackwardVectorizable: -; CHECK-NEXT: %l = load i32, ptr %gep, align 4 -> -; CHECK-NEXT: store i32 %add, ptr %gep.mul.2, align 4 -; CHECK-EMPTY: ; CHECK-NEXT: Run-time memory checks: ; CHECK-NEXT: Grouped accesses: ; CHECK-EMPTY: diff --git a/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll b/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll index 1a6e25859f085..468b22568277e 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/positive-dependence-distance-different-access-sizes.ll @@ -8,21 +8,10 @@ target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128" define void @test_distance_positive_independent_via_trip_count(ptr %A) { ; CHECK-LABEL: 'test_distance_positive_independent_via_trip_count' ; CHECK-NEXT: loop: -; CHECK-NEXT: Memory dependences are safe with run-time checks +; CHECK-NEXT: Memory dependences are safe ; CHECK-NEXT: Dependences: ; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Check 0: -; CHECK-NEXT: Comparing group GRP0: -; CHECK-NEXT: %gep.A.400 = getelementptr inbounds i32, ptr %A.400, i64 %iv -; CHECK-NEXT: Against group GRP1: -; CHECK-NEXT: %gep.A = getelementptr inbounds i8, ptr %A, i64 %iv ; CHECK-NEXT: Grouped accesses: -; CHECK-NEXT: Group GRP0: -; CHECK-NEXT: (Low: (400 + %A) High: (804 + %A)) -; CHECK-NEXT: Member: {(400 + %A),+,4}<%loop> -; CHECK-NEXT: Group GRP1: -; CHECK-NEXT: (Low: %A High: (101 + %A)) -; CHECK-NEXT: Member: {%A,+,1}<%loop> ; CHECK-EMPTY: ; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. ; CHECK-NEXT: SCEV assumptions: From ee153c8667d3d5e20dc0405cb0cb6599f3776ae1 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 25 Jul 2025 10:23:56 +0100 Subject: [PATCH 06/60] [LV] Add additional SCEV expansion tests for #147824. Add additional test coverage for https://github.com/llvm/llvm-project/pull/147824. (cherry picked from commit 6d004d2e5b69bb3eac33ff1fd8d6907c124abbc0) --- .../reuse-lcssa-phi-scev-expansion.ll | 212 ++++++++++++++++++ 1 file changed, 212 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll index ce4270dc4b7fa..1fc34db26261e 100644 --- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll @@ -194,3 +194,215 @@ loop.2.latch: exit: ret void } + + +declare void @foo() +declare void @bar() + +define void @expand_diff_scev_unknown(ptr %dst, i1 %invar.c, i32 %step) mustprogress { +; CHECK-LABEL: define void @expand_diff_scev_unknown( +; CHECK-SAME: ptr [[DST:%.*]], i1 [[INVAR_C:%.*]], i32 [[STEP:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_1:.*]] +; CHECK: [[LOOP_1]]: +; CHECK-NEXT: [[INDVAR:%.*]] = phi i32 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[IV_1:%.*]] = phi i32 [ [[STEP]], %[[ENTRY]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP_1]] ] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[IV_1_NEXT]] = add i32 [[IV_1]], 1 +; CHECK-NEXT: [[INDVAR_NEXT]] = add i32 [[INDVAR]], 1 +; CHECK-NEXT: br i1 [[INVAR_C]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_1]] +; CHECK: [[LOOP_2_PREHEADER]]: +; CHECK-NEXT: [[INDVAR_LCSSA1:%.*]] = phi i32 [ [[INDVAR]], %[[LOOP_1]] ] +; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i32 [ [[INDVAR]], %[[LOOP_1]] ] +; CHECK-NEXT: [[IV_1_LCSSA:%.*]] = phi i32 [ [[IV_1]], %[[LOOP_1]] ] +; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[STEP]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDVAR_LCSSA]], [[TMP0]] +; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0) +; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[STEP]], -2 +; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[INDVAR_LCSSA1]], -1 +; CHECK-NEXT: [[TMP4:%.*]] = add i32 [[TMP3]], [[TMP2]] +; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[SMAX]], [[TMP4]] +; CHECK-NEXT: [[UMIN:%.*]] = call i32 @llvm.umin.i32(i32 [[TMP5]], i32 1) +; CHECK-NEXT: [[TMP6:%.*]] = add i32 [[UMIN]], 1 +; CHECK-NEXT: [[TMP7:%.*]] = sub i32 [[TMP5]], [[UMIN]] +; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[STEP]], i32 1) +; CHECK-NEXT: [[TMP8:%.*]] = udiv i32 [[TMP7]], [[UMAX]] +; CHECK-NEXT: [[TMP9:%.*]] = add i32 [[TMP6]], [[TMP8]] +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP9]], 2 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_SCEVCHECK:.*]] +; CHECK: [[VECTOR_SCEVCHECK]]: +; CHECK-NEXT: [[IDENT_CHECK:%.*]] = icmp ne i32 [[STEP]], 1 +; CHECK-NEXT: br i1 [[IDENT_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP9]], 2 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP9]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP10:%.*]] = add i32 [[IV_1_LCSSA]], [[N_VEC]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[IV_1_LCSSA]], [[INDEX]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[OFFSET_IDX]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 +; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[TMP12]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 2 +; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP13]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP9]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[TMP10]], %[[MIDDLE_BLOCK]] ], [ [[IV_1_LCSSA]], %[[LOOP_2_PREHEADER]] ], [ [[IV_1_LCSSA]], %[[VECTOR_SCEVCHECK]] ] +; CHECK-NEXT: br label %[[LOOP_2:.*]] +; CHECK: [[LOOP_2]]: +; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[IV_2_NEXT]] = add nsw i32 [[IV_2]], [[STEP]] +; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds i32, ptr [[DST]], i32 [[IV_2]] +; CHECK-NEXT: store i32 0, ptr [[GEP_DST]], align 4 +; CHECK-NEXT: [[EC_2:%.*]] = icmp slt i32 [[IV_2_NEXT]], 0 +; CHECK-NEXT: br i1 [[EC_2]], label %[[LOOP_2]], label %[[EXIT]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.1 + +loop.1: + %iv.1 = phi i32 [ %step, %entry ], [ %iv.1.next, %loop.1 ] + call void @foo() + %iv.1.next = add i32 %iv.1, 1 + br i1 %invar.c, label %loop.2, label %loop.1 + +loop.2: + %iv.2 = phi i32 [ %iv.1, %loop.1 ], [ %iv.2.next, %loop.2 ] + %iv.2.next = add nsw i32 %iv.2, %step + %gep.dst = getelementptr inbounds i32, ptr %dst, i32 %iv.2 + store i32 0, ptr %gep.dst + %ec.2 = icmp slt i32 %iv.2.next, 0 + br i1 %ec.2, label %loop.2, label %exit + +exit: + ret void +} + +define void @expand_diff_neg_ptrtoint_expr(ptr %src, ptr %start) { +; CHECK-LABEL: define void @expand_diff_neg_ptrtoint_expr( +; CHECK-SAME: ptr [[SRC:%.*]], ptr [[START:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64 +; CHECK-NEXT: [[START1:%.*]] = ptrtoint ptr [[START]] to i64 +; CHECK-NEXT: br label %[[LOOP_1:.*]] +; CHECK: [[LOOP_1]]: +; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ] +; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ] +; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 8 +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i64 [[IV_NEXT]], 32 +; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 +; CHECK-NEXT: br i1 [[EC_1]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_1]] +; CHECK: [[LOOP_2_PREHEADER]]: +; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ] +; CHECK-NEXT: [[PTR_IV_1_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1_NEXT]], %[[LOOP_1]] ] +; CHECK-NEXT: br label %[[LOOP_2:.*]] +; CHECK: [[LOOP_2]]: +; CHECK-NEXT: [[INDVAR3:%.*]] = phi i64 [ 0, %[[LOOP_2_PREHEADER]] ], [ [[INDVAR_NEXT4:%.*]], %[[LOOP_2]] ] +; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[IV_NEXT_1:%.*]], %[[LOOP_2]] ], [ 1, %[[LOOP_2_PREHEADER]] ] +; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[LOOP_2_PREHEADER]] ] +; CHECK-NEXT: call void @bar() +; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 8 +; CHECK-NEXT: [[IV_NEXT_1]] = add i64 [[IV_1]], 1 +; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i64 [[IV_NEXT_1]], 32 +; CHECK-NEXT: [[INDVAR_NEXT4]] = add i64 [[INDVAR3]], 1 +; CHECK-NEXT: br i1 [[EC_2]], label %[[LOOP_3_PREHEADER:.*]], label %[[LOOP_2]] +; CHECK: [[LOOP_3_PREHEADER]]: +; CHECK-NEXT: [[INDVAR3_LCSSA:%.*]] = phi i64 [ [[INDVAR3]], %[[LOOP_2]] ] +; CHECK-NEXT: [[PTR_IV_2_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_2_NEXT]], %[[LOOP_2]] ] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[START1]], 16 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SRC2]] +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[INDVAR_LCSSA]], 3 +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], [[TMP1]] +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[INDVAR3_LCSSA]], 3 +; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], [[TMP3]] +; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], 16 +; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[PTR_IV_2_NEXT_LCSSA]], i64 -16 +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] +; CHECK-NEXT: [[OFFSET_IDX5:%.*]] = mul i64 [[INDEX]], 8 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_IV_2_NEXT_LCSSA]], i64 [[OFFSET_IDX5]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], -1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP9]], align 8 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: store <2 x i64> [[WIDE_LOAD]], ptr [[TMP10]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], -2 +; CHECK-NEXT: br i1 [[TMP11]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_3_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_2_NEXT_LCSSA]], %[[LOOP_3_PREHEADER]] ], [ [[PTR_IV_2_NEXT_LCSSA]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: br label %[[LOOP_3:.*]] +; CHECK: [[LOOP_3]]: +; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_NEXT_2:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[IV_2]], -1 +; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP12]] +; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP_SRC]], align 8 +; CHECK-NEXT: [[PTR_IV_3_NEXT]] = getelementptr i8, ptr [[PTR_IV_3]], i64 8 +; CHECK-NEXT: store i64 [[L]], ptr [[PTR_IV_3]], align 8 +; CHECK-NEXT: [[IV_NEXT_2]] = add i64 [[IV_2]], 1 +; CHECK-NEXT: [[EC_3:%.*]] = icmp eq i64 [[IV_NEXT_2]], 0 +; CHECK-NEXT: br i1 [[EC_3]], label %[[EXIT]], label %[[LOOP_3]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.1 + +loop.1: + %iv = phi i64 [ 1, %entry ], [ %iv.next, %loop.1 ] + %ptr.iv.1 = phi ptr [ %start, %entry ], [ %ptr.iv.1.next, %loop.1 ] + %ptr.iv.1.next = getelementptr i8, ptr %ptr.iv.1, i64 8 + call void @foo() + %iv.next = add i64 %iv, 1 + %ec.1 = icmp eq i64 %iv.next, 32 + br i1 %ec.1, label %loop.2, label %loop.1 + +loop.2: + %iv.1 = phi i64 [ 1, %loop.1 ], [ %iv.next.1, %loop.2 ] + %ptr.iv.2 = phi ptr [ %ptr.iv.1.next, %loop.1 ], [ %ptr.iv.2.next, %loop.2 ] + call void @bar() + %ptr.iv.2.next = getelementptr i8, ptr %ptr.iv.2, i64 8 + %iv.next.1 = add i64 %iv.1, 1 + %ec.2 = icmp eq i64 %iv.next.1, 32 + br i1 %ec.2, label %loop.3, label %loop.2 + +loop.3: + %iv.2 = phi i64 [ 1, %loop.2 ], [ %iv.next.2, %loop.3 ] + %ptr.iv.3 = phi ptr [ %ptr.iv.2.next, %loop.2 ], [ %ptr.iv.3.next, %loop.3 ] + %6 = add i64 %iv.2, -1 + %gep.src = getelementptr double, ptr %src, i64 %6 + %l = load i64, ptr %gep.src, align 8 + %ptr.iv.3.next = getelementptr i8, ptr %ptr.iv.3, i64 8 + store i64 %l, ptr %ptr.iv.3, align 8 + %iv.next.2 = add i64 %iv.2, 1 + %ec.3 = icmp eq i64 %iv.next.2, 0 + br i1 %ec.3, label %exit, label %loop.3 + +exit: + ret void +} + +; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) +declare double @llvm.cos.f64(double) #0 + +attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } From bcf033b1de66a6d19defc809feb49bf9dfd8d4d2 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 25 Jul 2025 15:29:40 +0100 Subject: [PATCH 07/60] [SCEV] Try to re-use pointer LCSSA phis when expanding SCEVs. (#147824) Generalize the code added in https://github.com/llvm/llvm-project/pull/147214 to also support re-using pointer LCSSA phis when expanding SCEVs with AddRecs. A common source of integer AddRecs with pointer bases are runtime checks emitted by LV based on the distance between 2 pointer AddRecs. This improves codegen in some cases when vectorizing and prevents regressions with https://github.com/llvm/llvm-project/pull/142309, which turns some phis into single-entry ones, which SCEV will look through now (and expand the whole AddRec), whereas before it would have to treat the LCSSA phi as SCEVUnknown. Compile-time impact neutral: https://llvm-compile-time-tracker.com/compare.php?from=fd5fc76c91538871771be2c3be2ca3a5f2dcac31&to=ca5fc2b3d8e6efc09f1624a17fdbfbe909f14eb4&stat=instructions:u PR: https://github.com/llvm/llvm-project/pull/147824 (cherry picked from commit e21ee41be450f849f5247aafa07d7f4c3941bb9d) --- .../Analysis/ScalarEvolutionPatternMatch.h | 6 ++ .../Utils/ScalarEvolutionExpander.cpp | 36 ++++++- .../reuse-lcssa-phi-scev-expansion.ll | 98 +++++++++++++++++++ .../invalidate-laa-after-versioning.ll | 9 +- .../reuse-lcssa-phi-scev-expansion.ll | 52 ++++------ .../invalidate-laa-after-versioning.ll | 13 +-- 6 files changed, 160 insertions(+), 54 deletions(-) create mode 100644 llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index 09e3945f5a8ff..bff77073ef4e1 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -160,6 +160,12 @@ m_scev_ZExt(const Op0_t &Op0) { return m_scev_Unary(Op0); } +template +inline SCEVUnaryExpr_match +m_scev_PtrToInt(const Op0_t &Op0) { + return SCEVUnaryExpr_match(Op0); +} + /// Match a binary SCEV. template struct SCEVBinaryExpr_match { diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index ace74d086e8db..ea8d136ee8680 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -18,6 +18,7 @@ #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolutionPatternMatch.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/DataLayout.h" @@ -42,6 +43,7 @@ cl::opt llvm::SCEVCheapExpansionBudget( "controls the budget that is considered cheap (default = 4)")); using namespace PatternMatch; +using namespace SCEVPatternMatch; PoisonFlags::PoisonFlags(const Instruction *I) { NUW = false; @@ -1224,6 +1226,7 @@ Value *SCEVExpander::expandAddRecExprLiterally(const SCEVAddRecExpr *S) { } Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) { + Type *STy = S->getType(); const Loop *L = S->getLoop(); BasicBlock *EB = L->getExitBlock(); if (!EB || !EB->getSinglePredecessor() || @@ -1231,11 +1234,36 @@ Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) { return nullptr; for (auto &PN : EB->phis()) { - if (!SE.isSCEVable(PN.getType()) || PN.getType() != S->getType()) + if (!SE.isSCEVable(PN.getType())) continue; - auto *ExitV = SE.getSCEV(&PN); - if (S == ExitV) - return &PN; + auto *ExitSCEV = SE.getSCEV(&PN); + if (!isa(ExitSCEV)) + continue; + Type *PhiTy = PN.getType(); + if (STy->isIntegerTy() && PhiTy->isPointerTy()) + ExitSCEV = SE.getPtrToIntExpr(ExitSCEV, STy); + else if (S->getType() != PN.getType()) + continue; + + // Check if we can re-use the existing PN, by adjusting it with an expanded + // offset, if the offset is simpler. + const SCEV *Diff = SE.getMinusSCEV(S, ExitSCEV); + const SCEV *Op = Diff; + match(Diff, m_scev_Mul(m_scev_AllOnes(), m_SCEV(Op))); + match(Op, m_scev_PtrToInt(m_SCEV(Op))); + if (!isa(Op)) + continue; + + assert(Diff->getType()->isIntegerTy() && + "difference must be of integer type"); + Value *DiffV = expand(Diff); + Value *BaseV = &PN; + if (PhiTy->isPointerTy()) { + if (STy->isPointerTy()) + return Builder.CreatePtrAdd(BaseV, DiffV); + BaseV = Builder.CreatePtrToInt(BaseV, DiffV->getType()); + } + return Builder.CreateAdd(BaseV, DiffV); } return nullptr; diff --git a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll new file mode 100644 index 0000000000000..357a7b6b7fd9a --- /dev/null +++ b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll @@ -0,0 +1,98 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -p loop-idiom -S %s | FileCheck %s + +declare void @foo() +declare void @bar() + +define void @scev_expand_ptrtoint(i8 %x, ptr %start) { +; CHECK-LABEL: define void @scev_expand_ptrtoint( +; CHECK-SAME: i8 [[X:%.*]], ptr [[START:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[START1:%.*]] = ptrtoint ptr [[START]] to i64 +; CHECK-NEXT: br label %[[LOOP_1_HEADER:.*]] +; CHECK: [[LOOP_1_HEADER]]: +; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1_LATCH:.*]] ] +; CHECK-NEXT: [[C:%.*]] = icmp ule i8 [[X]], 1 +; CHECK-NEXT: br i1 [[C]], label %[[LOOP_1_LATCH]], label %[[MIDDLE:.*]] +; CHECK: [[LOOP_1_LATCH]]: +; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 1 +; CHECK-NEXT: br label %[[LOOP_1_HEADER]] +; CHECK: [[MIDDLE]]: +; CHECK-NEXT: [[PTR_IV_1_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1]], %[[LOOP_1_HEADER]] ] +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[X]], 0 +; CHECK-NEXT: [[CMP_EXT:%.*]] = zext i1 [[CMP]] to i64 +; CHECK-NEXT: [[GEP_START:%.*]] = getelementptr i8, ptr [[PTR_IV_1_LCSSA]], i64 [[CMP_EXT]] +; CHECK-NEXT: br label %[[LOOP_2_HEADER:.*]] +; CHECK: [[LOOP_2_HEADER]]: +; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_2_LATCH:.*]] ], [ 0, %[[MIDDLE]] ] +; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[GEP_START]], %[[MIDDLE]] ], [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ] +; CHECK-NEXT: switch i8 [[X]], label %[[LOOP_2_LATCH]] [ +; CHECK-NEXT: i8 1, label %[[LOOP_3_PREHEADER:.*]] +; CHECK-NEXT: i8 4, label %[[LOOP_3_PREHEADER]] +; CHECK-NEXT: ] +; CHECK: [[LOOP_3_PREHEADER]]: +; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_2_HEADER]] ], [ [[INDVAR]], %[[LOOP_2_HEADER]] ] +; CHECK-NEXT: [[PTR_IV_2_LCSSA:%.*]] = phi ptr [ [[PTR_IV_2]], %[[LOOP_2_HEADER]] ], [ [[PTR_IV_2]], %[[LOOP_2_HEADER]] ] +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 0, [[START1]] +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR_IV_1_LCSSA]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[TMP0]] +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 1 +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[CMP_EXT]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDVAR_LCSSA]], [[TMP4]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]] +; CHECK-NEXT: [[STRLEN:%.*]] = call i64 @strlen(ptr [[SCEVGEP]]) +; CHECK-NEXT: br label %[[LOOP_3:.*]] +; CHECK: [[LOOP_2_LATCH]]: +; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 1 +; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 +; CHECK-NEXT: br label %[[LOOP_2_HEADER]] +; CHECK: [[LOOP_3]]: +; CHECK-NEXT: [[IV_3:%.*]] = phi i64 [ [[IV_3_NEXT:%.*]], %[[LOOP_3]] ], [ 1, %[[LOOP_3_PREHEADER]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[PTR_IV_2_LCSSA]], i64 [[IV_3]] +; CHECK-NEXT: [[TMP6:%.*]] = load i8, ptr [[GEP]], align 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i8 [[TMP6]], 0 +; CHECK-NEXT: [[IV_3_NEXT]] = add i64 [[IV_3]], 1 +; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[LOOP_3]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.1.header + +loop.1.header: + %ptr.iv.1 = phi ptr [ %start, %entry ], [ %ptr.iv.1.next, %loop.1.latch ] + %c = icmp ule i8 %x, 1 + br i1 %c, label %loop.1.latch, label %middle + +loop.1.latch: + %ptr.iv.1.next = getelementptr i8, ptr %ptr.iv.1, i64 1 + br label %loop.1.header + +middle: + %cmp = icmp eq i8 %x, 0 + %cmp.ext = zext i1 %cmp to i64 + %gep.start = getelementptr i8, ptr %ptr.iv.1, i64 %cmp.ext + br label %loop.2.header + +loop.2.header: + %ptr.iv.2 = phi ptr [ %gep.start, %middle ], [ %ptr.iv.2.next, %loop.2.latch ] + switch i8 %x, label %loop.2.latch [ + i8 1, label %loop.3 + i8 4, label %loop.3 + ] + +loop.2.latch: + %ptr.iv.2.next = getelementptr i8, ptr %ptr.iv.2, i64 1 + br label %loop.2.header + +loop.3: + %iv.3 = phi i64 [ 1, %loop.2.header ], [ 1, %loop.2.header ], [ %iv.3.next, %loop.3 ] + %gep = getelementptr i8, ptr %ptr.iv.2, i64 %iv.3 + %1 = load i8, ptr %gep, align 1 + %ec = icmp eq i8 %1, 0 + %iv.3.next = add i64 %iv.3, 1 + br i1 %ec, label %exit, label %loop.3 + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopLoadElim/invalidate-laa-after-versioning.ll b/llvm/test/Transforms/LoopLoadElim/invalidate-laa-after-versioning.ll index 10e10653a431d..037851f364e24 100644 --- a/llvm/test/Transforms/LoopLoadElim/invalidate-laa-after-versioning.ll +++ b/llvm/test/Transforms/LoopLoadElim/invalidate-laa-after-versioning.ll @@ -59,19 +59,14 @@ define void @test(ptr %arg, i64 %arg1) { ; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_1]], i64 1 ; CHECK-NEXT: br label [[INNER_2:%.*]] ; CHECK: inner.2: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[INNER_2]] ], [ 0, [[INNER_1_EXIT]] ] ; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[GEP_5]], [[INNER_1_EXIT]] ], [ [[PTR_IV_2_NEXT:%.*]], [[INNER_2]] ] ; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1 -; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 ; CHECK-NEXT: br i1 false, label [[INNER_3_LVER_CHECK:%.*]], label [[INNER_2]] ; CHECK: inner.3.lver.check: -; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], [[INNER_2]] ] ; CHECK-NEXT: [[LCSSA_PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2]], [[INNER_2]] ] ; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr inbounds double, ptr [[PTR_PHI]], i64 1 ; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_2]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INDVAR_LCSSA]], 3 -; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 24 -; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP1]] +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_2]], i64 16 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_7]], [[GEP_1]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR_PHI]], [[SCEVGEP3]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -104,7 +99,7 @@ define void @test(ptr %arg, i64 %arg1) { ; CHECK-NEXT: br i1 [[C_2]], label [[OUTER_LATCH_LOOPEXIT4:%.*]], label [[INNER_3]] ; CHECK: outer.latch.loopexit: ; CHECK-NEXT: br label [[OUTER_LATCH]] -; CHECK: outer.latch.loopexit4: +; CHECK: outer.latch.loopexit3: ; CHECK-NEXT: br label [[OUTER_LATCH]] ; CHECK: outer.latch: ; CHECK-NEXT: br label [[INNER_1_LVER_CHECK]] diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll index 1fc34db26261e..4697b4abf5ea8 100644 --- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll @@ -104,27 +104,23 @@ define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) { ; CHECK-LABEL: define void @runtime_checks_ptr_inductions( ; CHECK-SAME: ptr [[DST_1:%.*]], ptr [[DST_2:%.*]], i1 [[C:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: -; CHECK-NEXT: [[DST_11:%.*]] = ptrtoint ptr [[DST_1]] to i64 ; CHECK-NEXT: br label %[[LOOP_1:.*]] ; CHECK: [[LOOP_1]]: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ] ; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[DST_1]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ] ; CHECK-NEXT: [[CALL:%.*]] = call i32 @val() ; CHECK-NEXT: [[SEL_DST:%.*]] = select i1 [[C]], ptr [[DST_1]], ptr [[DST_2]] ; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 1 ; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i32 [[CALL]], 0 -; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 ; CHECK-NEXT: br i1 [[EC_1]], label %[[LOOP_2_HEADER_PREHEADER:.*]], label %[[LOOP_1]] ; CHECK: [[LOOP_2_HEADER_PREHEADER]]: -; CHECK-NEXT: [[SEL_DST_LCSSA2:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ] -; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ] +; CHECK-NEXT: [[SEL_DST_LCSSA1:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ] ; CHECK-NEXT: [[PTR_IV_1_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1]], %[[LOOP_1]] ] ; CHECK-NEXT: [[SEL_DST_LCSSA:%.*]] = phi ptr [ [[SEL_DST]], %[[LOOP_1]] ] -; CHECK-NEXT: [[SEL_DST_LCSSA23:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA2]] to i64 +; CHECK-NEXT: [[SEL_DST_LCSSA12:%.*]] = ptrtoint ptr [[SEL_DST_LCSSA1]] to i64 ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDVAR_LCSSA]], [[DST_11]] -; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SEL_DST_LCSSA23]] +; CHECK-NEXT: [[TMP0:%.*]] = ptrtoint ptr [[PTR_IV_1_LCSSA]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SEL_DST_LCSSA12]] ; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP1]], 2 ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: @@ -146,13 +142,13 @@ define void @runtime_checks_ptr_inductions(ptr %dst.1, ptr %dst.2, i1 %c) { ; CHECK-NEXT: br label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 1023, %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_2_HEADER_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL4:%.*]] = phi ptr [ [[TMP2]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[PTR_IV_1_LCSSA]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL5:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[SEL_DST_LCSSA]], %[[LOOP_2_HEADER_PREHEADER]] ], [ [[SEL_DST_LCSSA]], %[[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label %[[LOOP_2_HEADER:.*]] ; CHECK: [[LOOP_2_HEADER]]: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[DEC7:%.*]], %[[LOOP_2_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] -; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ] -; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL4]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_2_LATCH]] ], [ [[BC_RESUME_VAL5]], %[[SCALAR_PH]] ] ; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i32 [[IV]], 1024 ; CHECK-NEXT: br i1 [[EC_2]], label %[[EXIT:.*]], label %[[LOOP_2_LATCH]] ; CHECK: [[LOOP_2_LATCH]]: @@ -213,10 +209,8 @@ define void @expand_diff_scev_unknown(ptr %dst, i1 %invar.c, i32 %step) mustprog ; CHECK-NEXT: br i1 [[INVAR_C]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_1]] ; CHECK: [[LOOP_2_PREHEADER]]: ; CHECK-NEXT: [[INDVAR_LCSSA1:%.*]] = phi i32 [ [[INDVAR]], %[[LOOP_1]] ] -; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i32 [ [[INDVAR]], %[[LOOP_1]] ] ; CHECK-NEXT: [[IV_1_LCSSA:%.*]] = phi i32 [ [[IV_1]], %[[LOOP_1]] ] -; CHECK-NEXT: [[TMP0:%.*]] = shl i32 [[STEP]], 1 -; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[INDVAR_LCSSA]], [[TMP0]] +; CHECK-NEXT: [[TMP1:%.*]] = add i32 [[IV_1_LCSSA]], [[STEP]] ; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[TMP1]], i32 0) ; CHECK-NEXT: [[TMP2:%.*]] = mul i32 [[STEP]], -2 ; CHECK-NEXT: [[TMP3:%.*]] = mul i32 [[INDVAR_LCSSA1]], -1 @@ -289,53 +283,43 @@ define void @expand_diff_neg_ptrtoint_expr(ptr %src, ptr %start) { ; CHECK-SAME: ptr [[SRC:%.*]], ptr [[START:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64 -; CHECK-NEXT: [[START1:%.*]] = ptrtoint ptr [[START]] to i64 ; CHECK-NEXT: br label %[[LOOP_1:.*]] ; CHECK: [[LOOP_1]]: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ] ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_1]] ] ; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[START]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ] ; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 8 ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EC_1:%.*]] = icmp eq i64 [[IV_NEXT]], 32 -; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 ; CHECK-NEXT: br i1 [[EC_1]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_1]] ; CHECK: [[LOOP_2_PREHEADER]]: -; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ] ; CHECK-NEXT: [[PTR_IV_1_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1_NEXT]], %[[LOOP_1]] ] ; CHECK-NEXT: br label %[[LOOP_2:.*]] ; CHECK: [[LOOP_2]]: -; CHECK-NEXT: [[INDVAR3:%.*]] = phi i64 [ 0, %[[LOOP_2_PREHEADER]] ], [ [[INDVAR_NEXT4:%.*]], %[[LOOP_2]] ] ; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[IV_NEXT_1:%.*]], %[[LOOP_2]] ], [ 1, %[[LOOP_2_PREHEADER]] ] ; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[LOOP_2_PREHEADER]] ] ; CHECK-NEXT: call void @bar() ; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 8 ; CHECK-NEXT: [[IV_NEXT_1]] = add i64 [[IV_1]], 1 ; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i64 [[IV_NEXT_1]], 32 -; CHECK-NEXT: [[INDVAR_NEXT4]] = add i64 [[INDVAR3]], 1 ; CHECK-NEXT: br i1 [[EC_2]], label %[[LOOP_3_PREHEADER:.*]], label %[[LOOP_2]] ; CHECK: [[LOOP_3_PREHEADER]]: -; CHECK-NEXT: [[INDVAR3_LCSSA:%.*]] = phi i64 [ [[INDVAR3]], %[[LOOP_2]] ] -; CHECK-NEXT: [[PTR_IV_2_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_2_NEXT]], %[[LOOP_2]] ] +; CHECK-NEXT: [[TMP1:%.*]] = phi ptr [ [[PTR_IV_2_NEXT]], %[[LOOP_2]] ] ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[START1]], 16 -; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[SRC2]] -; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[INDVAR_LCSSA]], 3 -; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], [[TMP1]] -; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[INDVAR3_LCSSA]], 3 -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[TMP4]], [[TMP3]] -; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP5]], 16 +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 0, [[SRC2]] +; CHECK-NEXT: [[TMP5:%.*]] = ptrtoint ptr [[TMP1]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP5]], [[TMP0]] +; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], 16 ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i8, ptr [[PTR_IV_2_NEXT_LCSSA]], i64 -16 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[TMP1]], i64 -16 ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 1, [[INDEX]] ; CHECK-NEXT: [[OFFSET_IDX5:%.*]] = mul i64 [[INDEX]], 8 -; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_IV_2_NEXT_LCSSA]], i64 [[OFFSET_IDX5]] +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[TMP1]], i64 [[OFFSET_IDX5]] ; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[OFFSET_IDX]], -1 ; CHECK-NEXT: [[TMP8:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP7]] ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 @@ -349,11 +333,11 @@ define void @expand_diff_neg_ptrtoint_expr(ptr %src, ptr %start) { ; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ 1, %[[LOOP_3_PREHEADER]] ], [ 1, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL6:%.*]] = phi ptr [ [[TMP6]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_2_NEXT_LCSSA]], %[[LOOP_3_PREHEADER]] ], [ [[PTR_IV_2_NEXT_LCSSA]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[TMP1]], %[[LOOP_3_PREHEADER]] ], [ [[TMP1]], %[[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label %[[LOOP_3:.*]] ; CHECK: [[LOOP_3]]: ; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_NEXT_2:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ] -; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL6]], %[[SCALAR_PH]] ] +; CHECK-NEXT: [[PTR_IV_3:%.*]] = phi ptr [ [[PTR_IV_3_NEXT:%.*]], %[[LOOP_3]] ], [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ] ; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[IV_2]], -1 ; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr double, ptr [[SRC]], i64 [[TMP12]] ; CHECK-NEXT: [[L:%.*]] = load i64, ptr [[GEP_SRC]], align 8 diff --git a/llvm/test/Transforms/LoopVersioning/invalidate-laa-after-versioning.ll b/llvm/test/Transforms/LoopVersioning/invalidate-laa-after-versioning.ll index 8075314a65b49..4148c3541ae7a 100644 --- a/llvm/test/Transforms/LoopVersioning/invalidate-laa-after-versioning.ll +++ b/llvm/test/Transforms/LoopVersioning/invalidate-laa-after-versioning.ll @@ -56,19 +56,14 @@ define void @test(ptr %arg, i64 %arg1) { ; CHECK-NEXT: [[GEP_5:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_1]], i64 1 ; CHECK-NEXT: br label [[INNER_2:%.*]] ; CHECK: inner.2: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[INNER_2]] ], [ 0, [[INNER_1_EXIT]] ] ; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[GEP_5]], [[INNER_1_EXIT]] ], [ [[PTR_IV_2_NEXT:%.*]], [[INNER_2]] ] ; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr inbounds double, ptr [[PTR_IV_2]], i64 1 -; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 ; CHECK-NEXT: br i1 false, label [[INNER_3_LVER_CHECK:%.*]], label [[INNER_2]] ; CHECK: inner.3.lver.check: -; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], [[INNER_2]] ] ; CHECK-NEXT: [[LCSSA_PTR_IV_2:%.*]] = phi ptr [ [[PTR_IV_2]], [[INNER_2]] ] ; CHECK-NEXT: [[GEP_6:%.*]] = getelementptr inbounds double, ptr [[PTR_PHI]], i64 1 ; CHECK-NEXT: [[GEP_7:%.*]] = getelementptr inbounds double, ptr [[LCSSA_PTR_IV_2]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[INDVAR_LCSSA]], 3 -; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 24 -; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_1]], i64 [[TMP1]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[LCSSA_PTR_IV_2]], i64 16 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[GEP_7]], [[GEP_1]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR_PHI]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] @@ -90,10 +85,10 @@ define void @test(ptr %arg, i64 %arg1) { ; CHECK: inner.3: ; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ 0, [[INNER_3_PH]] ], [ [[IV_2_NEXT:%.*]], [[INNER_3]] ] ; CHECK-NEXT: [[GEP_8:%.*]] = getelementptr inbounds double, ptr [[GEP_6]], i64 [[IV_2]] -; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_7]], align 8, !alias.scope !0, !noalias !3 -; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_8]], align 8, !alias.scope !3 +; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_7]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]] +; CHECK-NEXT: store double 0.000000e+00, ptr [[GEP_8]], align 8, !alias.scope [[META3]] ; CHECK-NEXT: [[GEP_9:%.*]] = getelementptr double, ptr [[PTR_PHI]], i64 [[IV_2]] -; CHECK-NEXT: [[TMP18:%.*]] = load double, ptr [[GEP_9]], align 8, !alias.scope !3 +; CHECK-NEXT: [[TMP18:%.*]] = load double, ptr [[GEP_9]], align 8, !alias.scope [[META3]] ; CHECK-NEXT: [[IV_2_NEXT]] = add nuw nsw i64 [[IV_2]], 1 ; CHECK-NEXT: [[C_2:%.*]] = icmp eq i64 [[IV_2]], 1 ; CHECK-NEXT: br i1 [[C_2]], label [[OUTER_LATCH_LOOPEXIT3:%.*]], label [[INNER_3]] From 110e585a6cae3aec5b4717190aa4bbaba8346510 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 25 Jul 2025 21:08:39 +0100 Subject: [PATCH 08/60] [LV] Add test for re-using existing phi for SCEV Add. Add another test case for https://github.com/llvm/llvm-project/pull/147824, where the difference between an existing phi and the target SCEV is an add of a constant. (cherry picked from commit 445006d3a94e00bf6aa4ac650852aff52f0ca3b1) --- .../reuse-lcssa-phi-scev-expansion.ll | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll index 4697b4abf5ea8..fd32291123332 100644 --- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll @@ -386,6 +386,95 @@ exit: ret void } + +declare i1 @cond() + +define void @scev_exp_reuse_const_add(ptr %dst, ptr %src) { +; CHECK-LABEL: define void @scev_exp_reuse_const_add( +; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64 +; CHECK-NEXT: [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64 +; CHECK-NEXT: br label %[[LOOP_1:.*]] +; CHECK: [[LOOP_1]]: +; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ] +; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 2 +; CHECK-NEXT: [[C:%.*]] = call i1 @cond() +; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 +; CHECK-NEXT: br i1 [[C]], label %[[LOOP_2_PH:.*]], label %[[LOOP_1]] +; CHECK: [[LOOP_2_PH]]: +; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ] +; CHECK-NEXT: [[PTR_IV_1_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1_NEXT]], %[[LOOP_1]] ] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]] +; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[INDVAR_LCSSA]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[TMP0]] +; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 +; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[PTR_IV_1_NEXT_LCSSA]], i64 80 +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[PTR_IV_1_NEXT_LCSSA]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[TMP5]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i16>, ptr [[TMP6]], align 2 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: store <2 x i16> [[WIDE_LOAD]], ptr [[TMP7]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 40 +; CHECK-NEXT: br i1 [[TMP8]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 40, %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_2_PH]] ], [ 0, %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[LOOP_2_PH]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: br label %[[LOOP_2:.*]] +; CHECK: [[LOOP_2]]: +; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ] +; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2]] ] +; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_1]], 1 +; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV_2_NEXT]] +; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[GEP_SRC_1]], align 2 +; CHECK-NEXT: [[PTR_IV_2_NEXT]] = getelementptr i8, ptr [[PTR_IV_2]], i64 2 +; CHECK-NEXT: store i16 [[L]], ptr [[PTR_IV_2]], align 2 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_1]], 40 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT]], label %[[LOOP_2]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop.1 + +loop.1: + %ptr.iv.1 = phi ptr [ %dst, %entry ], [ %ptr.iv.1.next, %loop.1 ] + %ptr.iv.1.next = getelementptr i8, ptr %ptr.iv.1, i64 2 + %c = call i1 @cond() + br i1 %c, label %loop.2.ph, label %loop.1 + +loop.2.ph: + br label %loop.2 + +loop.2: + %iv.1 = phi i64 [ 0, %loop.2.ph ], [ %iv.2.next, %loop.2 ] + %ptr.iv.2 = phi ptr [ %ptr.iv.1.next, %loop.2.ph ], [ %ptr.iv.2.next, %loop.2 ] + %iv.2.next = add i64 %iv.1, 1 + %gep.src.1 = getelementptr i16, ptr %src, i64 %iv.2.next + %l = load i16, ptr %gep.src.1, align 2 + %ptr.iv.2.next = getelementptr i8, ptr %ptr.iv.2, i64 2 + store i16 %l, ptr %ptr.iv.2, align 2 + %ec = icmp eq i64 %iv.1, 40 + br i1 %ec, label %exit, label %loop.2 + +exit: + ret void +} + ; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none) declare double @llvm.cos.f64(double) #0 From 59fcf96e14a115e9036d78e488a5329b9b500fd0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 28 Jul 2025 16:24:46 +0100 Subject: [PATCH 09/60] [SCEV] Make sure LCSSA is preserved when re-using phi if needed. If we insert a new add instruction, it may introduce a new use outside the loop that contains the phi node we re-use. Use fixupLCSSAFormFor to fix LCSSA form, if needed. This fixes a crash reported in https://github.com/llvm/llvm-project/pull/147824#issuecomment-3124670997. (cherry picked from commit f9f68af4b8d5f8dd0bf8f14174b8b223fcf54c97) --- .../Utils/ScalarEvolutionExpander.cpp | 2 +- .../reuse-lcssa-phi-scev-expansion.ll | 66 +++++++++++ .../Hexagon/reuse-lcssa-phi-scev-expansion.ll | 108 ++++++++++++++++++ 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index ea8d136ee8680..a12e6f9b70aa2 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1257,7 +1257,7 @@ Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) { assert(Diff->getType()->isIntegerTy() && "difference must be of integer type"); Value *DiffV = expand(Diff); - Value *BaseV = &PN; + Value *BaseV = fixupLCSSAFormFor(&PN); if (PhiTy->isPointerTy()) { if (STy->isPointerTy()) return Builder.CreatePtrAdd(BaseV, DiffV); diff --git a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll index 357a7b6b7fd9a..919f9f242aa70 100644 --- a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll @@ -96,3 +96,69 @@ loop.3: exit: ret void } + +declare i1 @cond() + +define ptr @test_lcssa_reuse_preserve_lcssa() { +; CHECK-LABEL: define ptr @test_lcssa_reuse_preserve_lcssa() { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[LOOP_0_HEADER:.*]] +; CHECK: [[LOOP_0_HEADER]]: +; CHECK-NEXT: br label %[[LOOP_1:.*]] +; CHECK: [[LOOP_1]]: +; CHECK-NEXT: [[IV_1:%.*]] = phi ptr [ null, %[[LOOP_0_HEADER]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP_1]] ] +; CHECK-NEXT: [[IV_1_NEXT]] = getelementptr i8, ptr [[IV_1]], i64 1 +; CHECK-NEXT: [[EC_1:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[EC_1]], label %[[THEN:.*]], label %[[LOOP_1]] +; CHECK: [[THEN]]: +; CHECK-NEXT: [[IV_1_LCSSA1:%.*]] = phi ptr [ [[IV_1]], %[[LOOP_1]] ] +; CHECK-NEXT: [[C_2:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[C_2]], label %[[LOOP_2_PREHEADER:.*]], label %[[LOOP_0_LATCH:.*]] +; CHECK: [[LOOP_2_PREHEADER]]: +; CHECK-NEXT: [[IV_1_LCSSA:%.*]] = phi ptr [ [[IV_1_LCSSA1]], %[[THEN]] ] +; CHECK-NEXT: [[IV_1_LCSSA_LCSSA:%.*]] = phi ptr [ [[IV_1_LCSSA1]], %[[THEN]] ] +; CHECK-NEXT: [[STRLEN:%.*]] = call i64 @strlen(ptr null) +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr [[IV_1_LCSSA]], i64 1 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[TMP0]], i64 [[STRLEN]] +; CHECK-NEXT: br label %[[LOOP_2:.*]] +; CHECK: [[LOOP_2]]: +; CHECK-NEXT: [[IV_2:%.*]] = phi ptr [ [[RES:%.*]], %[[LOOP_2]] ], [ [[IV_1_LCSSA_LCSSA]], %[[LOOP_2_PREHEADER]] ] +; CHECK-NEXT: [[RES]] = getelementptr i8, ptr [[IV_2]], i64 1 +; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[IV_1_LCSSA_LCSSA]], align 1 +; CHECK-NEXT: [[EC_2:%.*]] = icmp eq i8 [[L]], 0 +; CHECK-NEXT: br i1 true, label %[[EXIT:.*]], label %[[LOOP_2]] +; CHECK: [[LOOP_0_LATCH]]: +; CHECK-NEXT: br label %[[LOOP_0_HEADER]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret ptr [[SCEVGEP]] +; +entry: + br label %loop.0.header + +loop.0.header: + br label %loop.1 + +loop.1: + %iv.1 = phi ptr [ null, %loop.0.header ], [ %iv.1.next, %loop.1 ] + %iv.1.next = getelementptr i8, ptr %iv.1, i64 1 + %ec.1 = call i1 @cond() + br i1 %ec.1, label %then, label %loop.1 + +then: + %c.2 = call i1 @cond() + br i1 %c.2, label %loop.2, label %loop.0.latch + +loop.2: + %iv.2 = phi ptr [ %res, %loop.2 ], [ %iv.1, %then ] + %res = getelementptr i8, ptr %iv.2, i64 1 + %l = load i8, ptr %iv.1, align 1 + %ec.2 = icmp eq i8 %l, 0 + br i1 %ec.2, label %exit, label %loop.2 + +loop.0.latch: + br label %loop.0.header + +exit: + ret ptr %res +} +>>>>>>> f9f68af4b8d5 ([SCEV] Make sure LCSSA is preserved when re-using phi if needed.) diff --git a/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll new file mode 100644 index 0000000000000..f74fb14e397f3 --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/Hexagon/reuse-lcssa-phi-scev-expansion.ll @@ -0,0 +1,108 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -p loop-unroll -S %s | FileCheck %s + +target triple = "hexagon-unknown-linux" + +declare void @foo() + +define void @preserve_lcssa_when_reusing_existing_phi() { +; CHECK-LABEL: define void @preserve_lcssa_when_reusing_existing_phi() { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[LOOP_1_HEADER:.*]] +; CHECK: [[LOOP_1_HEADER]]: +; CHECK-NEXT: br label %[[LOOP_2_HEADER:.*]] +; CHECK: [[LOOP_2_HEADER]]: +; CHECK-NEXT: br label %[[LOOP_3:.*]] +; CHECK: [[LOOP_3]]: +; CHECK-NEXT: [[IV_3:%.*]] = phi i32 [ [[IV_3_NEXT:%.*]], %[[LOOP_3]] ], [ 0, %[[LOOP_2_HEADER]] ] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[IV_3_NEXT]] = add i32 [[IV_3]], 1 +; CHECK-NEXT: br i1 false, label %[[PH:.*]], label %[[LOOP_3]] +; CHECK: [[PH]]: +; CHECK-NEXT: [[IV_3_LCSSA:%.*]] = phi i32 [ [[IV_3]], %[[LOOP_3]] ] +; CHECK-NEXT: br i1 true, label %[[LOOP_2_LATCH:.*]], label %[[LOOP_4_PREHEADER:.*]] +; CHECK: [[LOOP_4_PREHEADER]]: +; CHECK-NEXT: [[IV_3_LCSSA_LCSSA1:%.*]] = phi i32 [ [[IV_3_LCSSA]], %[[PH]] ] +; CHECK-NEXT: [[IV_3_LCSSA_LCSSA:%.*]] = phi i32 [ [[IV_3_LCSSA]], %[[PH]] ] +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[IV_3_LCSSA_LCSSA1]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = freeze i32 [[TMP0]] +; CHECK-NEXT: [[TMP2:%.*]] = add i32 [[TMP1]], -1 +; CHECK-NEXT: [[XTRAITER:%.*]] = and i32 [[TMP1]], 7 +; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i32 [[TMP2]], 7 +; CHECK-NEXT: br i1 [[TMP3]], label %[[LOOP_1_LATCH_UNR_LCSSA:.*]], label %[[LOOP_4_PREHEADER_NEW:.*]] +; CHECK: [[LOOP_4_PREHEADER_NEW]]: +; CHECK-NEXT: br label %[[LOOP_4:.*]] +; CHECK: [[LOOP_2_LATCH]]: +; CHECK-NEXT: br label %[[LOOP_2_HEADER]] +; CHECK: [[LOOP_4]]: +; CHECK-NEXT: [[IV_4:%.*]] = phi i32 [ 0, %[[LOOP_4_PREHEADER_NEW]] ], [ [[INC_I_7:%.*]], %[[LOOP_4]] ] +; CHECK-NEXT: [[NITER:%.*]] = phi i32 [ 0, %[[LOOP_4_PREHEADER_NEW]] ], [ [[NITER_NEXT_7:%.*]], %[[LOOP_4]] ] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[INC_I_7]] = add nuw nsw i32 [[IV_4]], 8 +; CHECK-NEXT: [[NITER_NEXT_7]] = add nuw nsw i32 [[NITER]], 8 +; CHECK-NEXT: br i1 true, label %[[LOOP_1_LATCH_UNR_LCSSA_LOOPEXIT:.*]], label %[[LOOP_4]] +; CHECK: [[LOOP_1_LATCH_UNR_LCSSA_LOOPEXIT]]: +; CHECK-NEXT: [[IV_4_UNR_PH:%.*]] = phi i32 [ [[INC_I_7]], %[[LOOP_4]] ] +; CHECK-NEXT: br label %[[LOOP_1_LATCH_UNR_LCSSA]] +; CHECK: [[LOOP_1_LATCH_UNR_LCSSA]]: +; CHECK-NEXT: [[IV_4_UNR:%.*]] = phi i32 [ 0, %[[LOOP_4_PREHEADER]] ], [ [[IV_4_UNR_PH]], %[[LOOP_1_LATCH_UNR_LCSSA_LOOPEXIT]] ] +; CHECK-NEXT: [[LCMP_MOD:%.*]] = icmp ne i32 [[XTRAITER]], 0 +; CHECK-NEXT: br i1 [[LCMP_MOD]], label %[[LOOP_4_EPIL_PREHEADER:.*]], label %[[LOOP_1_LATCH:.*]] +; CHECK: [[LOOP_4_EPIL_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP_4_EPIL:.*]] +; CHECK: [[LOOP_4_EPIL]]: +; CHECK-NEXT: [[IV_4_EPIL:%.*]] = phi i32 [ [[INC_I_EPIL:%.*]], %[[LOOP_4_EPIL]] ], [ [[IV_4_UNR]], %[[LOOP_4_EPIL_PREHEADER]] ] +; CHECK-NEXT: [[EPIL_ITER:%.*]] = phi i32 [ 0, %[[LOOP_4_EPIL_PREHEADER]] ], [ [[EPIL_ITER_NEXT:%.*]], %[[LOOP_4_EPIL]] ] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[INC_I_EPIL]] = add i32 [[IV_4_EPIL]], 1 +; CHECK-NEXT: [[EC_EPIL:%.*]] = icmp eq i32 [[IV_4_EPIL]], [[IV_3_LCSSA_LCSSA]] +; CHECK-NEXT: [[EPIL_ITER_NEXT]] = add i32 [[EPIL_ITER]], 1 +; CHECK-NEXT: [[EPIL_ITER_CMP:%.*]] = icmp ne i32 [[EPIL_ITER_NEXT]], [[XTRAITER]] +; CHECK-NEXT: br i1 [[EPIL_ITER_CMP]], label %[[LOOP_4_EPIL]], label %[[LOOP_1_LATCH_EPILOG_LCSSA:.*]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[LOOP_1_LATCH_EPILOG_LCSSA]]: +; CHECK-NEXT: br label %[[LOOP_1_LATCH]] +; CHECK: [[LOOP_1_LATCH]]: +; CHECK-NEXT: br label %[[LOOP_1_HEADER]] +; +entry: + br label %loop.1.header + +loop.1.header: + br label %loop.2.header + +loop.2.header: + br label %loop.3 + +loop.3: + %iv.3 = phi i32 [ %iv.3.next, %loop.3 ], [ 0, %loop.2.header ] + call void @foo() + %iv.3.next = add i32 %iv.3, 1 + br i1 false, label %ph, label %loop.3 + +ph: + br i1 true, label %loop.2.latch, label %loop.4 + +loop.2.latch: + br label %loop.2.header + +loop.4: + %iv.4 = phi i32 [ 0, %ph ], [ %inc.i, %loop.4 ] + call void @foo() + %inc.i = add i32 %iv.4, 1 + %ec = icmp eq i32 %iv.4, %iv.3 + br i1 %ec, label %loop.1.latch, label %loop.4 + +loop.1.latch: + br label %loop.1.header +} +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.unroll.disable"} +;. From 918407400dc32468fe391e8f2d2e2da38b541e64 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 30 Jul 2025 10:04:39 +0100 Subject: [PATCH 10/60] [SCEV] Add test for pushing constant add into zext. Adds a SCEV-only tests for https://github.com/llvm/llvm-project/pull/151227. (cherry picked from commit c6f7fa74376634619eb4e8ea9e9580fd3e220fe7) --- .../test/Analysis/ScalarEvolution/zext-add.ll | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 llvm/test/Analysis/ScalarEvolution/zext-add.ll diff --git a/llvm/test/Analysis/ScalarEvolution/zext-add.ll b/llvm/test/Analysis/ScalarEvolution/zext-add.ll new file mode 100644 index 0000000000000..3290ee2deb4e8 --- /dev/null +++ b/llvm/test/Analysis/ScalarEvolution/zext-add.ll @@ -0,0 +1,52 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes='print' -disable-output %s 2>&1 | FileCheck %s + +declare i1 @cond() + +define void @test_push_constant_into_zext(ptr %dst, ptr %src, i32 %n, i64 %offset) { +; CHECK-LABEL: 'test_push_constant_into_zext' +; CHECK-NEXT: Classifying expressions for: @test_push_constant_into_zext +; CHECK-NEXT: %outer.ptr = phi ptr [ %src, %entry ], [ %ptr.iv.next, %inner.loop ] +; CHECK-NEXT: --> %outer.ptr U: full-set S: full-set Exits: <> LoopDispositions: { %outer.loop: Variant, %inner.loop: Invariant } +; CHECK-NEXT: %c = call i1 @cond() +; CHECK-NEXT: --> %c U: full-set S: full-set Exits: <> LoopDispositions: { %outer.loop: Variant, %inner.loop: Invariant } +; CHECK-NEXT: %iv = phi i32 [ 0, %outer.loop ], [ %iv.next, %inner.loop ] +; CHECK-NEXT: --> {0,+,1}<%inner.loop> U: [0,2147483647) S: [0,2147483647) Exits: (-1 + (1 smax %n)) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } +; CHECK-NEXT: %ptr.iv = phi ptr [ %src, %outer.loop ], [ %ptr.iv.next, %inner.loop ] +; CHECK-NEXT: --> {%src,+,%offset}<%inner.loop> U: full-set S: full-set Exits: (((zext i32 (-1 + (1 smax %n)) to i64) * %offset) + %src) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } +; CHECK-NEXT: %l = load i8, ptr %outer.ptr, align 1 +; CHECK-NEXT: --> %l U: full-set S: full-set Exits: <> LoopDispositions: { %inner.loop: Variant, %outer.loop: Variant } +; CHECK-NEXT: %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 %offset +; CHECK-NEXT: --> {(%offset + %src),+,%offset}<%inner.loop> U: full-set S: full-set Exits: (((1 + (zext i32 (-1 + (1 smax %n)) to i64)) * %offset) + %src) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } +; CHECK-NEXT: %iv.next = add i32 %iv, 1 +; CHECK-NEXT: --> {1,+,1}<%inner.loop> U: [1,-2147483648) S: [1,-2147483648) Exits: (1 smax %n) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } +; CHECK-NEXT: Determining loop execution counts for: @test_push_constant_into_zext +; CHECK-NEXT: Loop %inner.loop: backedge-taken count is (-1 + (1 smax %n)) +; CHECK-NEXT: Loop %inner.loop: constant max backedge-taken count is i32 2147483646 +; CHECK-NEXT: Loop %inner.loop: symbolic max backedge-taken count is (-1 + (1 smax %n)) +; CHECK-NEXT: Loop %inner.loop: Trip multiple is 1 +; CHECK-NEXT: Loop %outer.loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %outer.loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %outer.loop: Unpredictable symbolic max backedge-taken count. +; +entry: + br label %outer.loop + +outer.loop: + %outer.ptr = phi ptr [ %src, %entry ], [ %ptr.iv.next, %inner.loop ] + %c = call i1 @cond() + br i1 %c, label %inner.loop, label %exit + +inner.loop: + %iv = phi i32 [ 0, %outer.loop ], [ %iv.next, %inner.loop ] + %ptr.iv = phi ptr [ %src, %outer.loop ], [ %ptr.iv.next, %inner.loop ] + %l = load i8, ptr %outer.ptr, align 1 + %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 %offset + store i8 %l, ptr %dst, align 2 + %iv.next = add i32 %iv, 1 + %ec = icmp slt i32 %iv.next, %n + br i1 %ec, label %inner.loop, label %outer.loop + +exit: + ret void +} From 4c941bf945002750f055dfd32d95a90731b8273e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 30 Jul 2025 21:10:57 +0100 Subject: [PATCH 11/60] [SECV] Try to push the op into ZExt: A + zext (-A + B) -> zext (B) (#151227) Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext (B), if trunc (A) + -A + B does not unsigned-wrap. The actual code supports ZExts with arbitrary number of arguments, hence the getAddExpr in the return. This helps SCEV reasoning in some cases, commonly when adding an offset to a zero-extended SCEV that subtracts the same offset. Note that this is restricted to cases where we can fold away an operand of the inner Add. This is needed to avoid bad interactions with patterns when forming ZExts, which try to push to ZExt to add operands. https://alive2.llvm.org/ce/z/q7d303 PR: https://github.com/llvm/llvm-project/pull/151227 (cherry picked from commit d74d841b65dc5ecc1adb87f94ee5c8073984e130) --- llvm/lib/Analysis/ScalarEvolution.cpp | 15 +++++ .../test/Analysis/ScalarEvolution/zext-add.ll | 2 +- .../IndVarSimplify/AArch64/fold-ext-add.ll | 55 +++++++++++++++++++ .../Transforms/IndVarSimplify/zext-nuw.ll | 6 +- .../reuse-lcssa-phi-scev-expansion.ll | 1 - 5 files changed, 73 insertions(+), 6 deletions(-) create mode 100644 llvm/test/Transforms/IndVarSimplify/AArch64/fold-ext-add.ll diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 1b6ebdd4eb111..cfa5dfdc94387 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -2682,6 +2682,21 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, return getAddExpr(NewOps, PreservedFlags); } } + + // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext + // (B), if trunc (A) + -A + B does not unsigned-wrap. + if (auto *ZExt = dyn_cast(Ops[1])) { + const SCEV *B = ZExt->getOperand(0); + const SCEV *NarrowA = getTruncateExpr(A, B->getType()); + if (isa(B) && + NarrowA == getNegativeSCEV(cast(B)->getOperand(0)) && + getZeroExtendExpr(NarrowA, ZExt->getType()) == A && + hasFlags(StrengthenNoWrapFlags(this, scAddExpr, {NarrowA, B}, + SCEV::FlagAnyWrap), + SCEV::FlagNUW)) { + return getZeroExtendExpr(getAddExpr(NarrowA, B), ZExt->getType()); + } + } } // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y) diff --git a/llvm/test/Analysis/ScalarEvolution/zext-add.ll b/llvm/test/Analysis/ScalarEvolution/zext-add.ll index 3290ee2deb4e8..a08feef7098ea 100644 --- a/llvm/test/Analysis/ScalarEvolution/zext-add.ll +++ b/llvm/test/Analysis/ScalarEvolution/zext-add.ll @@ -17,7 +17,7 @@ define void @test_push_constant_into_zext(ptr %dst, ptr %src, i32 %n, i64 %offse ; CHECK-NEXT: %l = load i8, ptr %outer.ptr, align 1 ; CHECK-NEXT: --> %l U: full-set S: full-set Exits: <> LoopDispositions: { %inner.loop: Variant, %outer.loop: Variant } ; CHECK-NEXT: %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 %offset -; CHECK-NEXT: --> {(%offset + %src),+,%offset}<%inner.loop> U: full-set S: full-set Exits: (((1 + (zext i32 (-1 + (1 smax %n)) to i64)) * %offset) + %src) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } +; CHECK-NEXT: --> {(%offset + %src),+,%offset}<%inner.loop> U: full-set S: full-set Exits: (((zext i32 (1 smax %n) to i64) * %offset) + %src) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } ; CHECK-NEXT: %iv.next = add i32 %iv, 1 ; CHECK-NEXT: --> {1,+,1}<%inner.loop> U: [1,-2147483648) S: [1,-2147483648) Exits: (1 smax %n) LoopDispositions: { %inner.loop: Computable, %outer.loop: Variant } ; CHECK-NEXT: Determining loop execution counts for: @test_push_constant_into_zext diff --git a/llvm/test/Transforms/IndVarSimplify/AArch64/fold-ext-add.ll b/llvm/test/Transforms/IndVarSimplify/AArch64/fold-ext-add.ll new file mode 100644 index 0000000000000..640c910be4a82 --- /dev/null +++ b/llvm/test/Transforms/IndVarSimplify/AArch64/fold-ext-add.ll @@ -0,0 +1,55 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -p indvars -S %s | FileCheck %s + +target triple = "arm64-apple-macosx15.0.0" + +declare i1 @cond() + +define void @pred_mip_12(ptr %dst, ptr %src, i32 %n, i64 %offset) { +; CHECK-LABEL: define void @pred_mip_12( +; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]], i32 [[N:%.*]], i64 [[OFFSET:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[N]], i32 1) +; CHECK-NEXT: [[TMP0:%.*]] = zext nneg i32 [[SMAX]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = mul i64 [[OFFSET]], [[TMP0]] +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[TMP1]] +; CHECK-NEXT: br label %[[OUTER_LOOP:.*]] +; CHECK: [[OUTER_LOOP_LOOPEXIT:.*]]: +; CHECK-NEXT: br label %[[OUTER_LOOP]] +; CHECK: [[OUTER_LOOP]]: +; CHECK-NEXT: [[OUTER_PTR:%.*]] = phi ptr [ [[SRC]], %[[ENTRY]] ], [ [[SCEVGEP]], %[[OUTER_LOOP_LOOPEXIT]] ] +; CHECK-NEXT: [[C:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[C]], label %[[INNER_LOOP_PREHEADER:.*]], label %[[EXIT:.*]] +; CHECK: [[INNER_LOOP_PREHEADER]]: +; CHECK-NEXT: br label %[[INNER_LOOP:.*]] +; CHECK: [[INNER_LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[INNER_LOOP]] ], [ 0, %[[INNER_LOOP_PREHEADER]] ] +; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[OUTER_PTR]], align 1 +; CHECK-NEXT: store i8 [[L]], ptr [[DST]], align 2 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[SMAX]] +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[INNER_LOOP]], label %[[OUTER_LOOP_LOOPEXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %outer.loop + +outer.loop: + %outer.ptr = phi ptr [ %src, %entry ], [ %ptr.iv.next, %inner.loop ] + %c = call i1 @cond() + br i1 %c, label %inner.loop, label %exit + +inner.loop: + %iv = phi i32 [ 0, %outer.loop ], [ %iv.next, %inner.loop ] + %ptr.iv = phi ptr [ %src, %outer.loop ], [ %ptr.iv.next, %inner.loop ] + %l = load i8, ptr %outer.ptr, align 1 + %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 %offset + store i8 %l, ptr %dst, align 2 + %iv.next = add i32 %iv, 1 + %ec = icmp slt i32 %iv.next, %n + br i1 %ec, label %inner.loop, label %outer.loop + +exit: + ret void +} diff --git a/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll b/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll index d24f9a4e40e38..17921afc5ff06 100644 --- a/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll +++ b/llvm/test/Transforms/IndVarSimplify/zext-nuw.ll @@ -15,11 +15,9 @@ define void @_Z3fn1v() { ; CHECK-NEXT: [[J_SROA_0_0_COPYLOAD:%.*]] = load i8, ptr [[X5]], align 1 ; CHECK-NEXT: br label [[DOTPREHEADER4_LR_PH:%.*]] ; CHECK: .preheader4.lr.ph: -; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[X4]], -1 -; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[TMP1]] to i64 -; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 ; CHECK-NEXT: [[TMP4:%.*]] = sext i8 [[J_SROA_0_0_COPYLOAD]] to i64 -; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[X4]] to i64 +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], [[TMP2]] ; CHECK-NEXT: br label [[DOTPREHEADER4:%.*]] ; CHECK: .preheader4: ; CHECK-NEXT: [[K_09:%.*]] = phi ptr [ undef, [[DOTPREHEADER4_LR_PH]] ], [ [[X25:%.*]], [[X22:%.*]] ] diff --git a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll index 919f9f242aa70..65aaf728b1ee8 100644 --- a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll @@ -161,4 +161,3 @@ loop.0.latch: exit: ret ptr %res } ->>>>>>> f9f68af4b8d5 ([SCEV] Make sure LCSSA is preserved when re-using phi if needed.) From e160af6b2bd0b9b530d45626e75ef7dd4ef37127 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 31 Jul 2025 12:47:13 +0100 Subject: [PATCH 12/60] [SCEV] Use pattern match to check ZExt(Add()). (NFC) Follow-up to https://github.com/llvm/llvm-project/pull/151227#pullrequestreview-3074670031 to check the inner expression is an Add before calling getTruncateExpr. Adds a new matcher that just matches and captures SCEVAddExpr, to support matching a SCEVAddExpr with arbitrary number of operands. (cherry picked from commit ab9b23c446531f5d9f081123c9f2fde4e8a334eb) --- .../llvm/Analysis/ScalarEvolutionPatternMatch.h | 4 ++++ llvm/lib/Analysis/ScalarEvolution.cpp | 15 +++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index bff77073ef4e1..011d5994dc670 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -91,6 +91,10 @@ inline bind_ty m_SCEVUnknown(const SCEVUnknown *&V) { return V; } +inline bind_ty m_scev_Add(const SCEVAddExpr *&V) { + return V; +} + /// Match a specified const SCEV *. struct specificscev_ty { const SCEV *Expr; diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index cfa5dfdc94387..4127ccb0c6f75 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -2685,16 +2685,15 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, // Try to push the constant operand into a ZExt: A + zext (-A + B) -> zext // (B), if trunc (A) + -A + B does not unsigned-wrap. - if (auto *ZExt = dyn_cast(Ops[1])) { - const SCEV *B = ZExt->getOperand(0); - const SCEV *NarrowA = getTruncateExpr(A, B->getType()); - if (isa(B) && - NarrowA == getNegativeSCEV(cast(B)->getOperand(0)) && - getZeroExtendExpr(NarrowA, ZExt->getType()) == A && - hasFlags(StrengthenNoWrapFlags(this, scAddExpr, {NarrowA, B}, + const SCEVAddExpr *InnerAdd; + if (match(B, m_scev_ZExt(m_scev_Add(InnerAdd)))) { + const SCEV *NarrowA = getTruncateExpr(A, InnerAdd->getType()); + if (NarrowA == getNegativeSCEV(InnerAdd->getOperand(0)) && + getZeroExtendExpr(NarrowA, B->getType()) == A && + hasFlags(StrengthenNoWrapFlags(this, scAddExpr, {NarrowA, InnerAdd}, SCEV::FlagAnyWrap), SCEV::FlagNUW)) { - return getZeroExtendExpr(getAddExpr(NarrowA, B), ZExt->getType()); + return getZeroExtendExpr(getAddExpr(NarrowA, InnerAdd), B->getType()); } } } From f0e4da7adb841d1f7afebcd7ea39179d8f39de27 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 31 Jul 2025 16:33:25 +0100 Subject: [PATCH 13/60] [SCEV] Allow adds of constants in tryToReuseLCSSAPhi. (#150693) Update the logic added in https://github.com/llvm/llvm-project/pull/147824 to also allow adds of constants. There are a number of cases where this can help remove redundant phis and replace some computation with a ptrtoint (which likely is free in the backend). PR: https://github.com/llvm/llvm-project/pull/150693 (cherry picked from commit 99d70e09a9676d63513a496f52acf93ca7167f00) --- .../lib/Transforms/Utils/ScalarEvolutionExpander.cpp | 3 ++- .../LoopIdiom/reuse-lcssa-phi-scev-expansion.ll | 5 ++--- .../LoopVectorize/reuse-lcssa-phi-scev-expansion.ll | 12 ++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index a12e6f9b70aa2..68e2cd44a776a 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1249,7 +1249,8 @@ Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) { // offset, if the offset is simpler. const SCEV *Diff = SE.getMinusSCEV(S, ExitSCEV); const SCEV *Op = Diff; - match(Diff, m_scev_Mul(m_scev_AllOnes(), m_SCEV(Op))); + match(Op, m_scev_Add(m_SCEVConstant(), m_SCEV(Op))); + match(Op, m_scev_Mul(m_scev_AllOnes(), m_SCEV(Op))); match(Op, m_scev_PtrToInt(m_SCEV(Op))); if (!isa(Op)) continue; diff --git a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll index 65aaf728b1ee8..cd3540143fdf9 100644 --- a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll @@ -33,11 +33,10 @@ define void @scev_expand_ptrtoint(i8 %x, ptr %start) { ; CHECK: [[LOOP_3_PREHEADER]]: ; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_2_HEADER]] ], [ [[INDVAR]], %[[LOOP_2_HEADER]] ] ; CHECK-NEXT: [[PTR_IV_2_LCSSA:%.*]] = phi ptr [ [[PTR_IV_2]], %[[LOOP_2_HEADER]] ], [ [[PTR_IV_2]], %[[LOOP_2_HEADER]] ] -; CHECK-NEXT: [[TMP0:%.*]] = sub i64 0, [[START1]] +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 1, [[START1]] ; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR_IV_1_LCSSA]] to i64 ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[TMP0]] -; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[TMP2]], 1 -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[CMP_EXT]], [[TMP3]] +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[CMP_EXT]], [[TMP2]] ; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDVAR_LCSSA]], [[TMP4]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP5]] ; CHECK-NEXT: [[STRLEN:%.*]] = call i64 @strlen(ptr [[SCEVGEP]]) diff --git a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll index fd32291123332..3eac7678bbc95 100644 --- a/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopVectorize/reuse-lcssa-phi-scev-expansion.ll @@ -394,22 +394,18 @@ define void @scev_exp_reuse_const_add(ptr %dst, ptr %src) { ; CHECK-SAME: ptr [[DST:%.*]], ptr [[SRC:%.*]]) { ; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64 -; CHECK-NEXT: [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64 ; CHECK-NEXT: br label %[[LOOP_1:.*]] ; CHECK: [[LOOP_1]]: -; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], %[[LOOP_1]] ], [ 0, %[[ENTRY]] ] ; CHECK-NEXT: [[PTR_IV_1:%.*]] = phi ptr [ [[DST]], %[[ENTRY]] ], [ [[PTR_IV_1_NEXT:%.*]], %[[LOOP_1]] ] ; CHECK-NEXT: [[PTR_IV_1_NEXT]] = getelementptr i8, ptr [[PTR_IV_1]], i64 2 ; CHECK-NEXT: [[C:%.*]] = call i1 @cond() -; CHECK-NEXT: [[INDVAR_NEXT]] = add i64 [[INDVAR]], 1 ; CHECK-NEXT: br i1 [[C]], label %[[LOOP_2_PH:.*]], label %[[LOOP_1]] ; CHECK: [[LOOP_2_PH]]: -; CHECK-NEXT: [[INDVAR_LCSSA:%.*]] = phi i64 [ [[INDVAR]], %[[LOOP_1]] ] ; CHECK-NEXT: [[PTR_IV_1_NEXT_LCSSA:%.*]] = phi ptr [ [[PTR_IV_1_NEXT]], %[[LOOP_1]] ] ; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: -; CHECK-NEXT: [[TMP0:%.*]] = sub i64 [[DST1]], [[SRC2]] -; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[INDVAR_LCSSA]], 1 +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 -2, [[SRC2]] +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[PTR_IV_1_NEXT_LCSSA]] to i64 ; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], [[TMP0]] ; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 ; CHECK-NEXT: br i1 [[DIFF_CHECK]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] @@ -433,11 +429,11 @@ define void @scev_exp_reuse_const_add(ptr %dst, ptr %src) { ; CHECK-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; CHECK: [[SCALAR_PH]]: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 40, %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_2_PH]] ], [ 0, %[[VECTOR_MEMCHECK]] ] -; CHECK-NEXT: [[BC_RESUME_VAL3:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[LOOP_2_PH]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: [[BC_RESUME_VAL2:%.*]] = phi ptr [ [[TMP3]], %[[MIDDLE_BLOCK]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[LOOP_2_PH]] ], [ [[PTR_IV_1_NEXT_LCSSA]], %[[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label %[[LOOP_2:.*]] ; CHECK: [[LOOP_2]]: ; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2]] ] -; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL3]], %[[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2]] ] +; CHECK-NEXT: [[PTR_IV_2:%.*]] = phi ptr [ [[BC_RESUME_VAL2]], %[[SCALAR_PH]] ], [ [[PTR_IV_2_NEXT:%.*]], %[[LOOP_2]] ] ; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_1]], 1 ; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr i16, ptr [[SRC]], i64 [[IV_2_NEXT]] ; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[GEP_SRC_1]], align 2 From a996352b3f2c8b473bbb4f23d123d6cb09d4c286 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 26 Aug 2025 11:38:39 +0100 Subject: [PATCH 14/60] [SCEVExp] Check if getPtrToIntExpr resulted in CouldNotCompute. This fixes a crash trying to use SCEVCouldNotCompute, if getPtrToIntExpr failed. Fixes https://github.com/llvm/llvm-project/issues/155287 (cherry-picked from b29084f0d80dd4fd66f1421350c87f79c537d071) --- .../Utils/ScalarEvolutionExpander.cpp | 7 ++- .../reuse-lcssa-phi-scev-expansion.ll | 62 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 68e2cd44a776a..e0b3a5b7892a1 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1240,10 +1240,13 @@ Value *SCEVExpander::tryToReuseLCSSAPhi(const SCEVAddRecExpr *S) { if (!isa(ExitSCEV)) continue; Type *PhiTy = PN.getType(); - if (STy->isIntegerTy() && PhiTy->isPointerTy()) + if (STy->isIntegerTy() && PhiTy->isPointerTy()) { ExitSCEV = SE.getPtrToIntExpr(ExitSCEV, STy); - else if (S->getType() != PN.getType()) + if (isa(ExitSCEV)) + continue; + } else if (S->getType() != PN.getType()) { continue; + } // Check if we can re-use the existing PN, by adjusting it with an expanded // offset, if the offset is simpler. diff --git a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll index cd3540143fdf9..a15db620e0082 100644 --- a/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll +++ b/llvm/test/Transforms/LoopIdiom/reuse-lcssa-phi-scev-expansion.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 ; RUN: opt -p loop-idiom -S %s | FileCheck %s +target datalayout = "e-m:e-i64:64-i128:128-f80:128-n8:16:32:64-S128-ni:1-p2:32:8:8:32-ni:2" + declare void @foo() declare void @bar() @@ -160,3 +162,63 @@ loop.0.latch: exit: ret ptr %res } + +; Test case for https://github.com/llvm/llvm-project/issues/155287. +; Make sure we don't crash when trying to construct a PtrToInt SCEV expression +; results in SCEVCouldNotCompute. +define void @phi_ptr_addressspace_ptrtoint_fail(ptr addrspace(1) %arg) { +; CHECK-LABEL: define void @phi_ptr_addressspace_ptrtoint_fail( +; CHECK-SAME: ptr addrspace(1) [[ARG:%.*]]) { +; CHECK-NEXT: [[BB:.*]]: +; CHECK-NEXT: br label %[[LOOP_1:.*]] +; CHECK: [[LOOP_1]]: +; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ 0, %[[BB]] ], [ [[ADD5:%.*]], %[[LOOP_1]] ] +; CHECK-NEXT: [[ADD5]] = add i64 [[IV_1]], 1 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i16, ptr addrspace(1) [[ARG]], i64 [[ADD5]] +; CHECK-NEXT: br i1 false, label %[[LOOP_1]], label %[[LOOP_2_PH:.*]] +; CHECK: [[LOOP_2_PH]]: +; CHECK-NEXT: [[IV_1_LCSSA1:%.*]] = phi i64 [ [[IV_1]], %[[LOOP_1]] ] +; CHECK-NEXT: [[IV_1_LCSSA:%.*]] = phi i64 [ [[IV_1]], %[[LOOP_1]] ] +; CHECK-NEXT: [[PHI:%.*]] = phi ptr addrspace(1) [ [[GETELEMENTPTR]], %[[LOOP_1]] ] +; CHECK-NEXT: [[TMP0:%.*]] = shl nuw nsw i64 [[IV_1_LCSSA1]], 2 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr addrspace(1) [[ARG]], i64 [[TMP0]] +; CHECK-NEXT: call void @llvm.memset.p1.i64(ptr addrspace(1) align 4 [[SCEVGEP]], i8 0, i64 8, i1 false) +; CHECK-NEXT: br label %[[LOOP_2_HEADER:.*]] +; CHECK: [[LOOP_2_HEADER]]: +; CHECK-NEXT: [[IV_2:%.*]] = phi i64 [ [[IV_1_LCSSA]], %[[LOOP_2_PH]] ], [ [[IV_2_NEXT:%.*]], %[[LOOP_2_LATCH:.*]] ] +; CHECK-NEXT: [[GREP_ARG:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i64 [[IV_2]] +; CHECK-NEXT: [[EC:%.*]] = icmp ult i64 [[IV_2]], 1 +; CHECK-NEXT: br i1 [[EC]], label %[[LOOP_2_LATCH]], label %[[EXIT:.*]] +; CHECK: [[LOOP_2_LATCH]]: +; CHECK-NEXT: [[IV_2_NEXT]] = add i64 [[IV_2]], 1 +; CHECK-NEXT: br label %[[LOOP_2_HEADER]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +bb: + br label %loop.1 + +loop.1: + %iv.1 = phi i64 [ 0, %bb ], [ %add5, %loop.1] + %add5 = add i64 %iv.1, 1 + %getelementptr = getelementptr i16, ptr addrspace(1) %arg, i64 %add5 + br i1 false, label %loop.1, label %loop.2.ph + +loop.2.ph: + %phi = phi ptr addrspace(1) [ %getelementptr, %loop.1] + br label %loop.2.header + +loop.2.header: + %iv.2 = phi i64 [ %iv.1, %loop.2.ph ], [ %iv.2.next, %loop.2.latch ] + %grep.arg = getelementptr i32, ptr addrspace(1) %arg, i64 %iv.2 + store i32 0, ptr addrspace(1) %grep.arg, align 4 + %ec = icmp ult i64 %iv.2, 1 + br i1 %ec, label %loop.2.latch, label %exit + +loop.2.latch: + %iv.2.next = add i64 %iv.2, 1 + br label %loop.2.header + +exit: + ret void +} From 0a096eb3ca379e15cfdb52ee9d072ed9a3d0b44e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 26 Aug 2025 19:31:50 +0100 Subject: [PATCH 15/60] [SCEV] Try to push op into ZExt: C * zext (A + B) -> zext (A*C + B*C) (#155300) Try to push constant multiply operand into a ZExt containing an add, if possible. In general we are trying to push down ops through ZExt if possible. This is similar to https://github.com/llvm/llvm-project/pull/151227 which did the same for additions. For now this is restricted to adds with a constant operand, which is similar to some of the logic above. This enables some additional simplifications. Alive2 Proof: https://alive2.llvm.org/ce/z/97pbSL PR: https://github.com/llvm/llvm-project/pull/155300 (cherry-picked from 914374624f19eb6c43bc272a4f600975f80e011a) --- llvm/lib/Analysis/ScalarEvolution.cpp | 15 +++++++++++++++ .../max-backedge-taken-count-guard-info.ll | 2 +- .../rewrite-loop-exit-values-phi.ll | 8 ++++---- .../LoopIdiom/X86/memset-size-compute.ll | 6 +++--- .../Transforms/LoopIdiom/add-nsw-zext-fold.ll | 6 +++--- .../Transforms/LoopVectorize/runtime-check.ll | 16 +++++++--------- 6 files changed, 33 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 4127ccb0c6f75..87191b6859f08 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3200,6 +3200,21 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, AddRec->getNoWrapFlags(FlagsMask)); } } + + // Try to push the constant operand into a ZExt: C * zext (A + B) -> + // zext (C*A + C*B) if trunc (C) * (A + B) does not unsigned-wrap. + const SCEVAddExpr *InnerAdd; + if (match(Ops[1], m_scev_ZExt(m_scev_Add(InnerAdd)))) { + const SCEV *NarrowC = getTruncateExpr(LHSC, InnerAdd->getType()); + if (isa(InnerAdd->getOperand(0)) && + getZeroExtendExpr(NarrowC, Ops[1]->getType()) == LHSC && + hasFlags(StrengthenNoWrapFlags(this, scMulExpr, {NarrowC, InnerAdd}, + SCEV::FlagAnyWrap), + SCEV::FlagNUW)) { + auto *Res = getMulExpr(NarrowC, InnerAdd, SCEV::FlagNUW, Depth + 1); + return getZeroExtendExpr(Res, Ops[1]->getType(), Depth + 1); + }; + } } } diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll index 9bf2427eddb9c..7cdf3a2d5fd58 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll @@ -1231,7 +1231,7 @@ define void @optimized_range_check_unsigned3(ptr %pred, i1 %c) { ; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] ; CHECK-NEXT: --> {0,+,1}<%loop> U: [0,3) S: [0,3) Exits: (-1 + %N) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i16, ptr %pred, i32 %iv -; CHECK-NEXT: --> {%pred,+,2}<%loop> U: full-set S: full-set Exits: ((2 * (zext i32 (-1 + %N) to i64)) + %pred) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%pred,+,2}<%loop> U: full-set S: full-set Exits: ((zext i32 (-2 + (2 * %N)) to i64) + %pred) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = add nuw nsw i32 %iv, 1 ; CHECK-NEXT: --> {1,+,1}<%loop> U: [1,4) S: [1,4) Exits: %N LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @optimized_range_check_unsigned3 diff --git a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll index e1a042747e0ac..84ae79d53e25e 100644 --- a/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll +++ b/llvm/test/Transforms/IndVarSimplify/rewrite-loop-exit-values-phi.ll @@ -76,18 +76,18 @@ define i64 @narow_canonical_iv_wide_multiplied_iv(i32 %x, i64 %y, ptr %0) { ; CHECK-LABEL: @narow_canonical_iv_wide_multiplied_iv( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SMAX:%.*]] = tail call i32 @llvm.smax.i32(i32 [[X:%.*]], i32 1) -; CHECK-NEXT: [[MUL_Y:%.*]] = shl i64 [[Y:%.*]], 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ] -; CHECK-NEXT: [[IV_MUL:%.*]] = phi i64 [ 1, [[ENTRY]] ], [ [[IV_MUL_NEXT:%.*]], [[LOOP]] ] -; CHECK-NEXT: [[IV_MUL_NEXT]] = add i64 [[IV_MUL]], [[MUL_Y]] ; CHECK-NEXT: call void @foo() ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp ne i32 [[IV_NEXT]], [[SMAX]] ; CHECK-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT:%.*]] ; CHECK: exit: -; CHECK-NEXT: [[TMP6:%.*]] = phi i64 [ [[IV_MUL_NEXT]], [[LOOP]] ] +; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[SMAX]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = mul i64 [[Y:%.*]], [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 1 +; CHECK-NEXT: [[TMP6:%.*]] = add nuw nsw i64 [[TMP3]], 1 ; CHECK-NEXT: ret i64 [[TMP6]] ; entry: diff --git a/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll b/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll index ea2cfe74be264..0123f15334281 100644 --- a/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll +++ b/llvm/test/Transforms/LoopIdiom/X86/memset-size-compute.ll @@ -16,10 +16,10 @@ define void @test(ptr %ptr) { ; CHECK-NEXT: [[LIM_0:%.*]] = phi i32 [ 65, [[ENTRY:%.*]] ], [ 1, [[DEAD:%.*]] ] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[PTR:%.*]], i64 8 ; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[LIM_0]], i32 2) -; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[UMAX]], -1 +; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i32 [[UMAX]], 3 +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i32 [[TMP2]], -8 ; CHECK-NEXT: [[TMP1:%.*]] = zext nneg i32 [[TMP0]] to i64 -; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 3 -; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 [[SCEVGEP]], i8 0, i64 [[TMP2]], i1 false) +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 [[SCEVGEP]], i8 0, i64 [[TMP1]], i1 false) ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[FOR_BODY]] ], [ 1, [[FOR_BODY_PREHEADER]] ] diff --git a/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll b/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll index df32e60d5065a..fc2b5571250ac 100644 --- a/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll +++ b/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll @@ -9,10 +9,10 @@ define void @fold_add_zext_to_sext(ptr %dst, i1 %start) { ; CHECK-NEXT: [[TMP0:%.*]] = zext i1 [[START]] to i64 ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]] -; CHECK-NEXT: [[TMP2:%.*]] = sub i32 25, [[START_EXT]] +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i32 [[START_EXT]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = sub i32 100, [[TMP4]] ; CHECK-NEXT: [[TMP3:%.*]] = zext nneg i32 [[TMP2]] to i64 -; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 -; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[SCEVGEP]], i8 0, i64 [[TMP4]], i1 false) +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 4 [[SCEVGEP]], i8 0, i64 [[TMP3]], i1 false) ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: ; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[START_EXT]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll index c05750ee83d92..0e9d3bdae5c6f 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll @@ -492,14 +492,12 @@ define void @test_scev_check_mul_add_expansion(ptr %out, ptr %in, i32 %len, i32 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]] ; CHECK: vector.memcheck: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[OUT:%.*]], i64 12 -; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[LEN]], -7 -; CHECK-NEXT: [[TMP2:%.*]] = zext nneg i32 [[TMP1]] to i64 -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i32 [[LEN]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = zext i32 [[TMP1]] to i64 ; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[OUT]], i64 [[TMP3]] -; CHECK-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[TMP4]], i64 14 ; CHECK-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[IN:%.*]], i64 4 ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[SCEVGEP]], [[SCEVGEP2]] -; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[IN]], [[SCEVGEP1]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[IN]], [[TMP4]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] ; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label [[SCALAR_PH]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: @@ -511,11 +509,11 @@ define void @test_scev_check_mul_add_expansion(ptr %out, ptr %in, i32 %len, i32 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[INDEX]], 6 ; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[OFFSET_IDX]] to i64 ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i16, ptr [[OUT]], i64 [[TMP6]] -; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP7]], align 2, !alias.scope [[META42:![0-9]+]], !noalias [[META45:![0-9]+]] -; CHECK-NEXT: store i32 0, ptr [[IN]], align 4, !alias.scope [[META45]] +; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr [[TMP7]], align 2, !alias.scope [[META37:![0-9]+]], !noalias [[META40:![0-9]+]] +; CHECK-NEXT: store i32 0, ptr [[IN]], align 4, !alias.scope [[META40]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP47:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP0]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] @@ -530,7 +528,7 @@ define void @test_scev_check_mul_add_expansion(ptr %out, ptr %in, i32 %len, i32 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 ; CHECK-NEXT: store i32 0, ptr [[IN]], align 4 ; CHECK-NEXT: [[CMP7_NOT:%.*]] = icmp sgt i32 [[LEN]], [[IV_NEXT]] -; CHECK-NEXT: br i1 [[CMP7_NOT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP48:![0-9]+]] +; CHECK-NEXT: br i1 [[CMP7_NOT]], label [[LOOP]], label [[EXIT]], !llvm.loop [[LOOP43:![0-9]+]] ; CHECK: exit: ; CHECK-NEXT: ret void ; From 83fdf694ee89501d9c8bbfef562edfe5c5b6bcaf Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 29 Aug 2025 13:14:58 +0100 Subject: [PATCH 16/60] [SCEV] Add tests for applying guards to SCEVAddExpr sub-expressions. Adds a test case for computing the backedge-taken-count for https://github.com/llvm/llvm-project/issues/155941 --- ...ge-taken-count-guard-info-apply-to-adds.ll | 29 +++++ ...ge-taken-count-guard-info-apply-to-adds.ll | 34 ++++++ .../test/Transforms/LoopVectorize/miniters.ll | 101 ++++++++++++++---- 3 files changed, 141 insertions(+), 23 deletions(-) create mode 100644 llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll create mode 100644 llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll new file mode 100644 index 0000000000000..ba859f2e3eec9 --- /dev/null +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll @@ -0,0 +1,29 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -passes='print' -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s + +define void @ptrtoint_based_trip_count_known_via_guards_applied_to_add_subexpr(ptr %start, ptr %end) { +; CHECK-LABEL: 'ptrtoint_based_trip_count_known_via_guards_applied_to_add_subexpr' +; CHECK-NEXT: Determining loop execution counts for: @ptrtoint_based_trip_count_known_via_guards_applied_to_add_subexpr +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %end.i = ptrtoint ptr %end to i64 + %start.i = ptrtoint ptr %start to i64 + %sub = sub i64 %end.i, %start.i + %pre.1 = icmp eq i64 %sub, 4 + call void @llvm.assume(i1 %pre.1) + br label %loop + +loop: + %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] + store i32 0, ptr %iv + %iv.next = getelementptr inbounds nuw i8, ptr %iv, i64 4 + %ec = icmp eq ptr %iv.next, %end + br i1 %ec, label %exit, label %loop + +exit: + ret void +} diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll new file mode 100644 index 0000000000000..635126c9262cf --- /dev/null +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 2 +; RUN: opt -passes='print' -scalar-evolution-classify-expressions=0 -disable-output %s 2>&1 | FileCheck %s + +define void @max_btc_improved_by_applying_guards_to_add_subexpr(i32 %low, i32 %high) { +; CHECK-LABEL: 'max_btc_improved_by_applying_guards_to_add_subexpr' +; CHECK-NEXT: Determining loop execution counts for: @max_btc_improved_by_applying_guards_to_add_subexpr +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 (1 + (-1 * %low) + %high) to i64)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 (1 + (-1 * %low) + %high) to i64)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %sub = sub i32 %high, %low + %pre.1 = icmp slt i32 %sub, 8 + br i1 %pre.1, label %if.then, label %exit + +if.then: + %pre.2 = icmp slt i32 %sub, 0 + br i1 %pre.2, label %exit, label %ph + +ph: + %add.1 = add i32 %sub, 1 + %wide.trip.count = zext i32 %add.1 to i64 + br label %loop + +loop: + %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop ] + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %wide.trip.count + br i1 %ec, label %exit, label %loop + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/miniters.ll b/llvm/test/Transforms/LoopVectorize/miniters.ll index 0b4c002045186..a0fd48d510f24 100644 --- a/llvm/test/Transforms/LoopVectorize/miniters.ll +++ b/llvm/test/Transforms/LoopVectorize/miniters.ll @@ -1,3 +1,4 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter-out-after "vector.ph:" --version 5 ; RUN: opt %s -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=4 -S | FileCheck %s ; RUN: opt %s -passes=loop-vectorize -force-vector-interleave=2 -force-vector-width=4 -S | FileCheck %s -check-prefix=UNROLL @@ -8,37 +9,91 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" @a = common global [1000 x i32] zeroinitializer, align 16 ; Generate min.iters.check to skip the vector loop and jump to scalar.ph directly when loop iteration number is less than VF * UF. -; CHECK-LABEL: foo( -; CHECK: %min.iters.check = icmp ult i64 %N, 4 -; CHECK: br i1 %min.iters.check, label %scalar.ph, label %vector.ph -; UNROLL-LABEL: foo( -; UNROLL: %min.iters.check = icmp ult i64 %N, 8 -; UNROLL: br i1 %min.iters.check, label %scalar.ph, label %vector.ph - define void @foo(i64 %N) { +; CHECK-LABEL: define void @foo( +; CHECK-SAME: i64 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[C:%.*]] = icmp sgt i64 [[N]], 0 +; CHECK-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; +; UNROLL-LABEL: define void @foo( +; UNROLL-SAME: i64 [[N:%.*]]) { +; UNROLL-NEXT: [[ENTRY:.*:]] +; UNROLL-NEXT: [[C:%.*]] = icmp sgt i64 [[N]], 0 +; UNROLL-NEXT: br i1 [[C]], label %[[LOOP_PREHEADER:.*]], [[EXIT:label %.*]] +; UNROLL: [[LOOP_PREHEADER]]: +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 8 +; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] +; UNROLL: [[VECTOR_PH]]: +; entry: - %cmp.8 = icmp sgt i64 %N, 0 - br i1 %cmp.8, label %for.body.preheader, label %for.end - -for.body.preheader: ; preds = %entry - br label %for.body + %c = icmp sgt i64 %N, 0 + br i1 %c, label %loop, label %exit -for.body: ; preds = %for.body, %for.body.preheader - %i.09 = phi i64 [ %inc, %for.body ], [ 0, %for.body.preheader ] - %arrayidx = getelementptr inbounds [1000 x i32], ptr @b, i64 0, i64 %i.09 +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + %arrayidx = getelementptr inbounds [1000 x i32], ptr @b, i64 0, i64 %iv %tmp = load i32, ptr %arrayidx, align 4 - %arrayidx1 = getelementptr inbounds [1000 x i32], ptr @c, i64 0, i64 %i.09 + %arrayidx1 = getelementptr inbounds [1000 x i32], ptr @c, i64 0, i64 %iv %tmp1 = load i32, ptr %arrayidx1, align 4 %add = add nsw i32 %tmp1, %tmp - %arrayidx2 = getelementptr inbounds [1000 x i32], ptr @a, i64 0, i64 %i.09 + %arrayidx2 = getelementptr inbounds [1000 x i32], ptr @a, i64 0, i64 %iv store i32 %add, ptr %arrayidx2, align 4 - %inc = add nuw nsw i64 %i.09, 1 - %exitcond = icmp eq i64 %inc, %N - br i1 %exitcond, label %for.end.loopexit, label %for.body + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %N + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @min_iters_known_via_loop_guards_add(i32 %start, i32 %end, ptr %src) { +; CHECK-LABEL: define void @min_iters_known_via_loop_guards_add( +; CHECK-SAME: i32 [[START:%.*]], i32 [[END:%.*]], ptr [[SRC:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[SUB:%.*]] = sub i32 [[END]], [[START]] +; CHECK-NEXT: [[PRE:%.*]] = icmp sgt i32 [[SUB]], 100 +; CHECK-NEXT: call void @llvm.assume(i1 [[PRE]]) +; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[SUB]], 1 +; CHECK-NEXT: [[IV_START:%.*]] = zext i32 [[ADD_1]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = sub i64 101, [[IV_START]] +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; +; UNROLL-LABEL: define void @min_iters_known_via_loop_guards_add( +; UNROLL-SAME: i32 [[START:%.*]], i32 [[END:%.*]], ptr [[SRC:%.*]]) { +; UNROLL-NEXT: [[ENTRY:.*:]] +; UNROLL-NEXT: [[SUB:%.*]] = sub i32 [[END]], [[START]] +; UNROLL-NEXT: [[PRE:%.*]] = icmp sgt i32 [[SUB]], 100 +; UNROLL-NEXT: call void @llvm.assume(i1 [[PRE]]) +; UNROLL-NEXT: [[ADD_1:%.*]] = add i32 [[SUB]], 1 +; UNROLL-NEXT: [[IV_START:%.*]] = zext i32 [[ADD_1]] to i64 +; UNROLL-NEXT: [[TMP0:%.*]] = sub i64 101, [[IV_START]] +; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8 +; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] +; UNROLL: [[VECTOR_PH]]: +; +entry: + %sub = sub i32 %end, %start + %pre = icmp sgt i32 %sub, 100 + call void @llvm.assume(i1 %pre) + %add.1 = add i32 %sub, 1 + %iv.start = zext i32 %add.1 to i64 + br label %loop -for.end.loopexit: ; preds = %for.body - br label %for.end +loop: + %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i64, ptr %src, i64 %iv + store i64 %iv, ptr %gep + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, 100 + br i1 %ec, label %exit, label %loop -for.end: ; preds = %for.end.loopexit, %entry +exit: ret void } From 9703b9841a18b3cd0dab18654917a0bcf3182efd Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 1 Sep 2025 14:01:15 +0100 Subject: [PATCH 17/60] [SCEV] Rewrite some SCEVAdd sub-expressions using loop guards. (#156013) Trip count expressions sometimes consist of adding 3 operands, i.e. (Const + A + B). There may be guard info for A + B, and if so, apply it. We can probably more generally apply this, but need to be careful w.r.t compile-time. Alive2 Proof for changes in miniters.ll: https://alive2.llvm.org/ce/z/HFfXOx Fixes https://github.com/llvm/llvm-project/issues/155941. PR: https://github.com/llvm/llvm-project/pull/156013 (cherry-picked from fb7c0d70a7aa284e1ef509e415849b4eea992ec9) --- llvm/lib/Analysis/ScalarEvolution.cpp | 10 ++++++++++ .../backedge-taken-count-guard-info-apply-to-adds.ll | 6 +++--- ...ax-backedge-taken-count-guard-info-apply-to-adds.ll | 2 +- llvm/test/Transforms/LoopVectorize/miniters.ll | 6 ++---- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 87191b6859f08..8b7d0e338e5ef 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15975,6 +15975,16 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const { } const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { + // Trip count expressions sometimes consist of adding 3 operands, i.e. + // (Const + A + B). There may be guard info for A + B, and if so, apply + // it. + // TODO: Could more generally apply guards to Add sub-expressions. + if (isa(Expr->getOperand(0)) && + Expr->getNumOperands() == 3) { + if (const SCEV *S = Map.lookup( + SE.getAddExpr(Expr->getOperand(1), Expr->getOperand(2)))) + return SE.getAddExpr(Expr->getOperand(0), S); + } SmallVector Operands; bool Changed = false; for (const auto *Op : Expr->operands()) { diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll index ba859f2e3eec9..75014f3a58eb6 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll @@ -4,9 +4,9 @@ define void @ptrtoint_based_trip_count_known_via_guards_applied_to_add_subexpr(ptr %start, ptr %end) { ; CHECK-LABEL: 'ptrtoint_based_trip_count_known_via_guards_applied_to_add_subexpr' ; CHECK-NEXT: Determining loop execution counts for: @ptrtoint_based_trip_count_known_via_guards_applied_to_add_subexpr -; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: backedge-taken count is i64 0 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 0 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is i64 0 ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll index 635126c9262cf..951b07272dd4b 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-apply-to-adds.ll @@ -5,7 +5,7 @@ define void @max_btc_improved_by_applying_guards_to_add_subexpr(i32 %low, i32 %h ; CHECK-LABEL: 'max_btc_improved_by_applying_guards_to_add_subexpr' ; CHECK-NEXT: Determining loop execution counts for: @max_btc_improved_by_applying_guards_to_add_subexpr ; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 (1 + (-1 * %low) + %high) to i64)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 7 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 (1 + (-1 * %low) + %high) to i64)) ; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; diff --git a/llvm/test/Transforms/LoopVectorize/miniters.ll b/llvm/test/Transforms/LoopVectorize/miniters.ll index a0fd48d510f24..6d06a03d0d018 100644 --- a/llvm/test/Transforms/LoopVectorize/miniters.ll +++ b/llvm/test/Transforms/LoopVectorize/miniters.ll @@ -61,8 +61,7 @@ define void @min_iters_known_via_loop_guards_add(i32 %start, i32 %end, ptr %src) ; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[SUB]], 1 ; CHECK-NEXT: [[IV_START:%.*]] = zext i32 [[ADD_1]] to i64 ; CHECK-NEXT: [[TMP0:%.*]] = sub i64 101, [[IV_START]] -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 false, [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; ; UNROLL-LABEL: define void @min_iters_known_via_loop_guards_add( @@ -74,8 +73,7 @@ define void @min_iters_known_via_loop_guards_add(i32 %start, i32 %end, ptr %src) ; UNROLL-NEXT: [[ADD_1:%.*]] = add i32 [[SUB]], 1 ; UNROLL-NEXT: [[IV_START:%.*]] = zext i32 [[ADD_1]] to i64 ; UNROLL-NEXT: [[TMP0:%.*]] = sub i64 101, [[IV_START]] -; UNROLL-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 8 -; UNROLL-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] +; UNROLL-NEXT: br i1 false, [[SCALAR_PH:label %.*]], label %[[VECTOR_PH:.*]] ; UNROLL: [[VECTOR_PH]]: ; entry: From 2ed995140fef7a45e69b42b319d0e0d2cac6a1d5 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Tue, 22 Jul 2025 13:36:33 +0100 Subject: [PATCH 18/60] [LAA] Rename var used to retry with RT-checks (NFC) (#147307) FoundNonConstantDistanceDependence is a misleading name for a variable that determines whether we retry with runtime checks. Rename it. (cherry picked from commit b692b239f037ebdc5a4501884a7f57a19afa33fd) --- .../llvm/Analysis/LoopAccessAnalysis.h | 10 ++++---- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 23 +++++++++---------- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 73bfe1aabb4e0..af6e534983709 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -236,8 +236,8 @@ class MemoryDepChecker { /// In same cases when the dependency check fails we can still /// vectorize the loop with a dynamic array access check. - bool shouldRetryWithRuntimeCheck() const { - return FoundNonConstantDistanceDependence && + bool shouldRetryWithRuntimeChecks() const { + return ShouldRetryWithRuntimeChecks && Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks; } @@ -327,9 +327,9 @@ class MemoryDepChecker { uint64_t MaxStoreLoadForwardSafeDistanceInBits = std::numeric_limits::max(); - /// If we see a non-constant dependence distance we can still try to - /// vectorize this loop with runtime checks. - bool FoundNonConstantDistanceDependence = false; + /// Whether we should try to vectorize the loop with runtime checks, if the + /// dependencies are not safe. + bool ShouldRetryWithRuntimeChecks = false; /// Result of the dependence checks, indicating whether the checked /// dependences are safe for vectorization, require RT checks or are known to diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index f8f741575f87a..062a89766d332 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -589,11 +589,11 @@ void RuntimePointerChecking::groupChecks( // dependence. Not grouping the checks for a[i] and a[i + 9000] allows // us to perform an accurate check in this case. // - // The above case requires that we have an UnknownDependence between - // accesses to the same underlying object. This cannot happen unless - // FoundNonConstantDistanceDependence is set, and therefore UseDependencies - // is also false. In this case we will use the fallback path and create - // separate checking groups for all pointers. + // In the above case, we have a non-constant distance and an Unknown + // dependence between accesses to the same underlying object, and could retry + // with runtime checks. Therefore UseDependencies is false. In this case we + // will use the fallback path and create separate checking groups for all + // pointers. // If we don't have the dependency partitions, construct a new // checking pointer group for each pointer. This is also required @@ -819,7 +819,7 @@ class AccessAnalysis { /// perform dependency checking. /// /// Note that this can later be cleared if we retry memcheck analysis without - /// dependency checking (i.e. FoundNonConstantDistanceDependence). + /// dependency checking (i.e. ShouldRetryWithRuntimeChecks). bool isDependencyCheckNeeded() const { return !CheckDeps.empty(); } /// We decided that no dependence analysis would be used. Reset the state. @@ -896,7 +896,7 @@ class AccessAnalysis { /// /// Note that, this is different from isDependencyCheckNeeded. When we retry /// memcheck analysis without dependency checking - /// (i.e. FoundNonConstantDistanceDependence), isDependencyCheckNeeded is + /// (i.e. ShouldRetryWithRuntimeChecks), isDependencyCheckNeeded is /// cleared while this remains set if we have potentially dependent accesses. bool IsRTCheckAnalysisNeeded = false; @@ -2079,11 +2079,10 @@ MemoryDepChecker::getDependenceDistanceStrideAndSize( if (StrideAScaled == StrideBScaled) CommonStride = StrideAScaled; - // TODO: FoundNonConstantDistanceDependence is used as a necessary condition - // to consider retrying with runtime checks. Historically, we did not set it - // when (unscaled) strides were different but there is no inherent reason to. + // TODO: Historically, we didn't retry with runtime checks when (unscaled) + // strides were different but there is no inherent reason to. if (!isa(Dist)) - FoundNonConstantDistanceDependence |= StrideAPtrInt == StrideBPtrInt; + ShouldRetryWithRuntimeChecks |= StrideAPtrInt == StrideBPtrInt; return DepDistanceStrideAndSizeInfo(Dist, MaxStride, CommonStride, TypeByteSize, AIsWrite, BIsWrite); @@ -2713,7 +2712,7 @@ bool LoopAccessInfo::analyzeLoop(AAResults *AA, const LoopInfo *LI, DepsAreSafe = DepChecker->areDepsSafe(DepCands, Accesses.getDependenciesToCheck()); - if (!DepsAreSafe && DepChecker->shouldRetryWithRuntimeCheck()) { + if (!DepsAreSafe && DepChecker->shouldRetryWithRuntimeChecks()) { LLVM_DEBUG(dbgs() << "LAA: Retrying with memory checks\n"); // Clear the dependency checks. We assume they are not needed. From 68d4d9e65f2682a6bd3e9fd2b864b50a83869b2c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 1 Aug 2025 14:18:07 +0100 Subject: [PATCH 19/60] [LAA] Support assumptions in evaluatePtrAddRecAtMaxBTCWillNotWrap (#147047) This patch extends the logic added in https://github.com/llvm/llvm-project/pull/128061 to support dereferenceability information from assumptions as well. Unfortunately both assumption cache and the dominator tree need to be threaded through multiple layers to make them available where needed. PR: https://github.com/llvm/llvm-project/pull/147047 (cherry picked from commit 2ae996cbbe92f006d711eeeb8f29e0946fc1c1e8) --- .../llvm/Analysis/LoopAccessAnalysis.h | 29 +++++-- llvm/lib/Analysis/Loads.cpp | 2 +- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 76 ++++++++++++------- llvm/lib/Transforms/Scalar/LoopFlatten.cpp | 3 +- .../Transforms/Scalar/LoopVersioningLICM.cpp | 2 +- .../early-exit-runtime-checks.ll | 8 +- .../single-early-exit-deref-assumptions.ll | 33 +++++++- .../Transforms/Vectorize/VPlanSlpTest.cpp | 3 +- 8 files changed, 111 insertions(+), 45 deletions(-) diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index af6e534983709..92304edd67a44 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -180,10 +180,12 @@ class MemoryDepChecker { const SmallVectorImpl &Instrs) const; }; - MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L, + MemoryDepChecker(PredicatedScalarEvolution &PSE, AssumptionCache *AC, + DominatorTree *DT, const Loop *L, const DenseMap &SymbolicStrides, unsigned MaxTargetVectorWidthInBits) - : PSE(PSE), InnermostLoop(L), SymbolicStrides(SymbolicStrides), + : PSE(PSE), AC(AC), DT(DT), InnermostLoop(L), + SymbolicStrides(SymbolicStrides), MaxTargetVectorWidthInBits(MaxTargetVectorWidthInBits) {} /// Register the location (instructions are given increasing numbers) @@ -288,6 +290,15 @@ class MemoryDepChecker { return PointerBounds; } + DominatorTree *getDT() const { + assert(DT && "requested DT, but it is not available"); + return DT; + } + AssumptionCache *getAC() const { + assert(AC && "requested AC, but it is not available"); + return AC; + } + private: /// A wrapper around ScalarEvolution, used to add runtime SCEV checks, and /// applies dynamic knowledge to simplify SCEV expressions and convert them @@ -296,6 +307,10 @@ class MemoryDepChecker { /// example we might assume a unit stride for a pointer in order to prove /// that a memory access is strided and doesn't wrap. PredicatedScalarEvolution &PSE; + + AssumptionCache *AC; + DominatorTree *DT; + const Loop *InnermostLoop; /// Reference to map of pointer values to @@ -670,7 +685,7 @@ class LoopAccessInfo { LLVM_ABI LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, AAResults *AA, - DominatorTree *DT, LoopInfo *LI, + DominatorTree *DT, LoopInfo *LI, AssumptionCache *AC, bool AllowPartial = false); /// Return true we can analyze the memory accesses in the loop and there are @@ -922,7 +937,8 @@ LLVM_ABI std::pair getStartAndEndForAccess( const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, const SCEV *BTC, const SCEV *MaxBTC, ScalarEvolution *SE, DenseMap, - std::pair> *PointerBounds); + std::pair> *PointerBounds, + DominatorTree *DT, AssumptionCache *AC); class LoopAccessInfoManager { /// The cache. @@ -935,12 +951,13 @@ class LoopAccessInfoManager { LoopInfo &LI; TargetTransformInfo *TTI; const TargetLibraryInfo *TLI = nullptr; + AssumptionCache *AC; public: LoopAccessInfoManager(ScalarEvolution &SE, AAResults &AA, DominatorTree &DT, LoopInfo &LI, TargetTransformInfo *TTI, - const TargetLibraryInfo *TLI) - : SE(SE), AA(AA), DT(DT), LI(LI), TTI(TTI), TLI(TLI) {} + const TargetLibraryInfo *TLI, AssumptionCache *AC) + : SE(SE), AA(AA), DT(DT), LI(LI), TTI(TTI), TLI(TLI), AC(AC) {} LLVM_ABI const LoopAccessInfo &getInfo(Loop &L, bool AllowPartial = false); diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index fcc2cf2f7e8e7..ee8b37feeca66 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -342,7 +342,7 @@ bool llvm::isDereferenceableAndAlignedInLoop( : SE.getConstantMaxBackedgeTakenCount(L); } const auto &[AccessStart, AccessEnd] = getStartAndEndForAccess( - L, PtrScev, LI->getType(), BECount, MaxBECount, &SE, nullptr); + L, PtrScev, LI->getType(), BECount, MaxBECount, &SE, nullptr, &DT, AC); if (isa(AccessStart) || isa(AccessEnd)) return false; diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 062a89766d332..c1c2c1009d0fa 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -23,6 +23,8 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/AliasSetTracker.h" +#include "llvm/Analysis/AssumeBundleQueries.h" +#include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/LoopAnalysisManager.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/LoopIterator.h" @@ -208,28 +210,46 @@ static const SCEV *mulSCEVOverflow(const SCEV *A, const SCEV *B, /// Return true, if evaluating \p AR at \p MaxBTC cannot wrap, because \p AR at /// \p MaxBTC is guaranteed inbounds of the accessed object. -static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, - const SCEV *MaxBTC, - const SCEV *EltSize, - ScalarEvolution &SE, - const DataLayout &DL) { +static bool +evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, + const SCEV *MaxBTC, const SCEV *EltSize, + ScalarEvolution &SE, const DataLayout &DL, + DominatorTree *DT, AssumptionCache *AC) { auto *PointerBase = SE.getPointerBase(AR->getStart()); auto *StartPtr = dyn_cast(PointerBase); if (!StartPtr) return false; + const Loop *L = AR->getLoop(); bool CheckForNonNull, CheckForFreed; - uint64_t DerefBytes = StartPtr->getValue()->getPointerDereferenceableBytes( + Value *StartPtrV = StartPtr->getValue(); + uint64_t DerefBytes = StartPtrV->getPointerDereferenceableBytes( DL, CheckForNonNull, CheckForFreed); - if (CheckForNonNull || CheckForFreed) + if (DerefBytes && (CheckForNonNull || CheckForFreed)) return false; const SCEV *Step = AR->getStepRecurrence(SE); + Type *WiderTy = SE.getWiderType(MaxBTC->getType(), Step->getType()); + const SCEV *DerefBytesSCEV = SE.getConstant(WiderTy, DerefBytes); + + // Check if we have a suitable dereferencable assumption we can use. + if (!StartPtrV->canBeFreed()) { + RetainedKnowledge DerefRK = getKnowledgeValidInContext( + StartPtrV, {Attribute::Dereferenceable}, *AC, + L->getLoopPredecessor()->getTerminator(), DT); + if (DerefRK) { + DerefBytesSCEV = SE.getUMaxExpr( + DerefBytesSCEV, SE.getConstant(WiderTy, DerefRK.ArgValue)); + } + } + + if (DerefBytesSCEV->isZero()) + return false; + bool IsKnownNonNegative = SE.isKnownNonNegative(Step); if (!IsKnownNonNegative && !SE.isKnownNegative(Step)) return false; - Type *WiderTy = SE.getWiderType(MaxBTC->getType(), Step->getType()); Step = SE.getNoopOrSignExtend(Step, WiderTy); MaxBTC = SE.getNoopOrZeroExtend(MaxBTC, WiderTy); @@ -256,8 +276,7 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, const SCEV *EndBytes = addSCEVNoOverflow(StartOffset, OffsetEndBytes, SE); if (!EndBytes) return false; - return SE.isKnownPredicate(CmpInst::ICMP_ULE, EndBytes, - SE.getConstant(WiderTy, DerefBytes)); + return SE.isKnownPredicate(CmpInst::ICMP_ULE, EndBytes, DerefBytesSCEV); } // For negative steps check if @@ -265,15 +284,15 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, // * StartOffset <= DerefBytes. assert(SE.isKnownNegative(Step) && "must be known negative"); return SE.isKnownPredicate(CmpInst::ICMP_SGE, StartOffset, OffsetEndBytes) && - SE.isKnownPredicate(CmpInst::ICMP_ULE, StartOffset, - SE.getConstant(WiderTy, DerefBytes)); + SE.isKnownPredicate(CmpInst::ICMP_ULE, StartOffset, DerefBytesSCEV); } std::pair llvm::getStartAndEndForAccess( const Loop *Lp, const SCEV *PtrExpr, Type *AccessTy, const SCEV *BTC, const SCEV *MaxBTC, ScalarEvolution *SE, DenseMap, - std::pair> *PointerBounds) { + std::pair> *PointerBounds, + DominatorTree *DT, AssumptionCache *AC) { std::pair *PtrBoundsPair; if (PointerBounds) { auto [Iter, Ins] = PointerBounds->insert( @@ -308,8 +327,8 @@ std::pair llvm::getStartAndEndForAccess( // sets ScEnd to the maximum unsigned value for the type. Note that LAA // separately checks that accesses cannot not wrap, so unsigned max // represents an upper bound. - if (evaluatePtrAddRecAtMaxBTCWillNotWrap(AR, MaxBTC, EltSizeSCEV, *SE, - DL)) { + if (evaluatePtrAddRecAtMaxBTCWillNotWrap(AR, MaxBTC, EltSizeSCEV, *SE, DL, + DT, AC)) { ScEnd = AR->evaluateAtIteration(MaxBTC, *SE); } else { ScEnd = SE->getAddExpr( @@ -356,9 +375,9 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, bool NeedsFreeze) { const SCEV *SymbolicMaxBTC = PSE.getSymbolicMaxBackedgeTakenCount(); const SCEV *BTC = PSE.getBackedgeTakenCount(); - const auto &[ScStart, ScEnd] = - getStartAndEndForAccess(Lp, PtrExpr, AccessTy, BTC, SymbolicMaxBTC, - PSE.getSE(), &DC.getPointerBounds()); + const auto &[ScStart, ScEnd] = getStartAndEndForAccess( + Lp, PtrExpr, AccessTy, BTC, SymbolicMaxBTC, PSE.getSE(), + &DC.getPointerBounds(), DC.getDT(), DC.getAC()); assert(!isa(ScStart) && !isa(ScEnd) && "must be able to compute both start and end expressions"); @@ -1961,13 +1980,15 @@ bool MemoryDepChecker::areAccessesCompletelyBeforeOrAfter(const SCEV *Src, const SCEV *BTC = PSE.getBackedgeTakenCount(); const SCEV *SymbolicMaxBTC = PSE.getSymbolicMaxBackedgeTakenCount(); ScalarEvolution &SE = *PSE.getSE(); - const auto &[SrcStart_, SrcEnd_] = getStartAndEndForAccess( - InnermostLoop, Src, SrcTy, BTC, SymbolicMaxBTC, &SE, &PointerBounds); + const auto &[SrcStart_, SrcEnd_] = + getStartAndEndForAccess(InnermostLoop, Src, SrcTy, BTC, SymbolicMaxBTC, + &SE, &PointerBounds, DT, AC); if (isa(SrcStart_) || isa(SrcEnd_)) return false; - const auto &[SinkStart_, SinkEnd_] = getStartAndEndForAccess( - InnermostLoop, Sink, SinkTy, BTC, SymbolicMaxBTC, &SE, &PointerBounds); + const auto &[SinkStart_, SinkEnd_] = + getStartAndEndForAccess(InnermostLoop, Sink, SinkTy, BTC, SymbolicMaxBTC, + &SE, &PointerBounds, DT, AC); if (isa(SinkStart_) || isa(SinkEnd_)) return false; @@ -3003,7 +3024,7 @@ LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE, const TargetTransformInfo *TTI, const TargetLibraryInfo *TLI, AAResults *AA, DominatorTree *DT, LoopInfo *LI, - bool AllowPartial) + AssumptionCache *AC, bool AllowPartial) : PSE(std::make_unique(*SE, *L)), PtrRtChecking(nullptr), TheLoop(L), AllowPartial(AllowPartial) { unsigned MaxTargetVectorWidthInBits = std::numeric_limits::max(); @@ -3013,8 +3034,8 @@ LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE, MaxTargetVectorWidthInBits = TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector) * 2; - DepChecker = std::make_unique(*PSE, L, SymbolicStrides, - MaxTargetVectorWidthInBits); + DepChecker = std::make_unique( + *PSE, AC, DT, L, SymbolicStrides, MaxTargetVectorWidthInBits); PtrRtChecking = std::make_unique(*DepChecker, SE); if (canAnalyzeLoop()) CanVecMem = analyzeLoop(AA, LI, TLI, DT); @@ -3083,7 +3104,7 @@ const LoopAccessInfo &LoopAccessInfoManager::getInfo(Loop &L, // or if it was created with a different value of AllowPartial. if (Inserted || It->second->hasAllowPartial() != AllowPartial) It->second = std::make_unique(&L, &SE, TTI, TLI, &AA, &DT, - &LI, AllowPartial); + &LI, AC, AllowPartial); return *It->second; } @@ -3126,7 +3147,8 @@ LoopAccessInfoManager LoopAccessAnalysis::run(Function &F, auto &LI = FAM.getResult(F); auto &TTI = FAM.getResult(F); auto &TLI = FAM.getResult(F); - return LoopAccessInfoManager(SE, AA, DT, LI, &TTI, &TLI); + auto &AC = FAM.getResult(F); + return LoopAccessInfoManager(SE, AA, DT, LI, &TTI, &TLI, &AC); } AnalysisKey LoopAccessAnalysis::Key; diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp index f3e992c039178..04039b885f3c5 100644 --- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp +++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp @@ -1009,7 +1009,8 @@ PreservedAnalyses LoopFlattenPass::run(LoopNest &LN, LoopAnalysisManager &LAM, // in simplified form, and also needs LCSSA. Running // this pass will simplify all loops that contain inner loops, // regardless of whether anything ends up being flattened. - LoopAccessInfoManager LAIM(AR.SE, AR.AA, AR.DT, AR.LI, &AR.TTI, nullptr); + LoopAccessInfoManager LAIM(AR.SE, AR.AA, AR.DT, AR.LI, &AR.TTI, nullptr, + &AR.AC); for (Loop *InnerLoop : LN.getLoops()) { auto *OuterLoop = InnerLoop->getParentLoop(); if (!OuterLoop) diff --git a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp index 4f2bfb073bafa..448dc2b8b52b0 100644 --- a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp +++ b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp @@ -551,7 +551,7 @@ PreservedAnalyses LoopVersioningLICMPass::run(Loop &L, LoopAnalysisManager &AM, const Function *F = L.getHeader()->getParent(); OptimizationRemarkEmitter ORE(F); - LoopAccessInfoManager LAIs(*SE, *AA, *DT, LAR.LI, nullptr, nullptr); + LoopAccessInfoManager LAIs(*SE, *AA, *DT, LAR.LI, nullptr, nullptr, &LAR.AC); if (!LoopVersioningLICM(AA, SE, &ORE, LAIs, LAR.LI, &L).run(DT)) return PreservedAnalyses::all(); return getLoopPassPreservedAnalyses(); diff --git a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll index 1dc8d4a7e73f8..207a44d5d08d4 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll @@ -505,7 +505,7 @@ e.1: ret i32 1 } -define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption(ptr %A, ptr %B) { +define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption(ptr %A, ptr %B) nosync nofree { ; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption' ; CHECK-NEXT: loop.header: ; CHECK-NEXT: Memory dependences are safe with run-time checks @@ -518,10 +518,10 @@ define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_kno ; CHECK-NEXT: %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv ; CHECK-NEXT: Grouped accesses: ; CHECK-NEXT: Group GRP0: -; CHECK-NEXT: (Low: %B High: inttoptr (i64 -1 to ptr)) +; CHECK-NEXT: (Low: %B High: (2000 + %B)) ; CHECK-NEXT: Member: {%B,+,4}<%loop.header> ; CHECK-NEXT: Group GRP1: -; CHECK-NEXT: (Low: %A High: inttoptr (i64 -1 to ptr)) +; CHECK-NEXT: (Low: %A High: (2000 + %A)) ; CHECK-NEXT: Member: {%A,+,4}<%loop.header> ; CHECK-EMPTY: ; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. @@ -565,7 +565,7 @@ e.2: ret void } -define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small(ptr %A, ptr %B) { +define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small(ptr %A, ptr %B) nosync nofree { ; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_deref_via_assumption_too_small' ; CHECK-NEXT: loop.header: ; CHECK-NEXT: Memory dependences are safe with run-time checks diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index e79995f673fb4..f329a18f3eaea 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -7,21 +7,46 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_si ; CHECK-NEXT: entry: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ] +; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: ; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX1]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX1]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1 +; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] +; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024 +; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] +; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: br label [[LOOP_END:%.*]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true) +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], [[TMP8]] +; CHECK-NEXT: br label [[LOOP_END]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: br label [[LOOP1:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]] ; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 ; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]] ; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1 ; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] -; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]] +; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]] ; CHECK: loop.inc: ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024 -; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP1]], label [[LOOP_END]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: loop.end: -; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ -1, [[LOOP_INC]] ] +; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP1]] ], [ -1, [[LOOP_INC]] ], [ -1, [[MIDDLE_BLOCK]] ], [ [[TMP9]], [[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: ret i64 [[RETVAL]] ; entry: diff --git a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp index 118bf67320a3b..7471355603640 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanSlpTest.cpp @@ -41,7 +41,8 @@ class VPlanSlpTest : public VPlanTestIRBase { AARes.reset(new AAResults(*TLI)); AARes->addAAResult(*BasicAA); PSE.reset(new PredicatedScalarEvolution(*SE, *L)); - LAI.reset(new LoopAccessInfo(L, &*SE, nullptr, &*TLI, &*AARes, &*DT, &*LI)); + LAI.reset( + new LoopAccessInfo(L, &*SE, nullptr, &*TLI, &*AARes, &*DT, &*LI, &*AC)); IAI.reset(new InterleavedAccessInfo(*PSE, L, &*DT, &*LI, &*LAI)); IAI->analyzeInterleaving(false); return {Plan, *IAI}; From 75a5fcdac27c3c45657b9c1b1172055c45089c8b Mon Sep 17 00:00:00 2001 From: annamthomas Date: Wed, 27 Aug 2025 08:11:32 -0400 Subject: [PATCH 20/60] [SCEV][LAA] Support multiplication overflow computation (#155236) Add support for identifying multiplication overflow in SCEV. This is needed in LoopAccessAnalysis and that limitation was worked around by 484417a. This allows early-exit vectorization to work as expected in vect.stats.ll test without needing the workaround. (cherry picked from commit 00926a6db6b3379aac05bf6b180123d1e2b0d382) --- llvm/lib/Analysis/Loads.cpp | 7 +++++- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 22 +++++++++++-------- llvm/lib/Analysis/ScalarEvolution.cpp | 14 +++++++++--- .../Transforms/LoopVectorize/vect.stats.ll | 2 +- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index ee8b37feeca66..94af3cc5861c3 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -26,6 +26,10 @@ using namespace llvm; +static cl::opt + UseSymbolicMaxBTCForDerefInLoop("use-symbolic-maxbtc-deref-loop", + cl::init(false)); + static bool isAligned(const Value *Base, Align Alignment, const DataLayout &DL) { return Base->getPointerAlignment(DL) >= Alignment; @@ -333,7 +337,7 @@ bool llvm::isDereferenceableAndAlignedInLoop( if (isa(MaxBECount)) return false; - if (isa(BECount)) { + if (isa(BECount) && !UseSymbolicMaxBTCForDerefInLoop) { // TODO: Support symbolic max backedge taken counts for loops without // computable backedge taken counts. MaxBECount = @@ -341,6 +345,7 @@ bool llvm::isDereferenceableAndAlignedInLoop( ? SE.getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates) : SE.getConstantMaxBackedgeTakenCount(L); } + const auto &[AccessStart, AccessEnd] = getStartAndEndForAccess( L, PtrScev, LI->getType(), BECount, MaxBECount, &SE, nullptr, &DT, AC); if (isa(AccessStart) || diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index c1c2c1009d0fa..cf4c5785f1e26 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -193,8 +193,9 @@ RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup( /// Returns \p A + \p B, if it is guaranteed not to unsigned wrap. Otherwise /// return nullptr. \p A and \p B must have the same type. static const SCEV *addSCEVNoOverflow(const SCEV *A, const SCEV *B, - ScalarEvolution &SE) { - if (!SE.willNotOverflow(Instruction::Add, /*IsSigned=*/false, A, B)) + ScalarEvolution &SE, + const Instruction *CtxI) { + if (!SE.willNotOverflow(Instruction::Add, /*IsSigned=*/false, A, B, CtxI)) return nullptr; return SE.getAddExpr(A, B); } @@ -202,8 +203,9 @@ static const SCEV *addSCEVNoOverflow(const SCEV *A, const SCEV *B, /// Returns \p A * \p B, if it is guaranteed not to unsigned wrap. Otherwise /// return nullptr. \p A and \p B must have the same type. static const SCEV *mulSCEVOverflow(const SCEV *A, const SCEV *B, - ScalarEvolution &SE) { - if (!SE.willNotOverflow(Instruction::Mul, /*IsSigned=*/false, A, B)) + ScalarEvolution &SE, + const Instruction *CtxI) { + if (!SE.willNotOverflow(Instruction::Mul, /*IsSigned=*/false, A, B, CtxI)) return nullptr; return SE.getMulExpr(A, B); } @@ -232,11 +234,12 @@ evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, Type *WiderTy = SE.getWiderType(MaxBTC->getType(), Step->getType()); const SCEV *DerefBytesSCEV = SE.getConstant(WiderTy, DerefBytes); + // Context which dominates the entire loop. + auto *CtxI = L->getLoopPredecessor()->getTerminator(); // Check if we have a suitable dereferencable assumption we can use. if (!StartPtrV->canBeFreed()) { RetainedKnowledge DerefRK = getKnowledgeValidInContext( - StartPtrV, {Attribute::Dereferenceable}, *AC, - L->getLoopPredecessor()->getTerminator(), DT); + StartPtrV, {Attribute::Dereferenceable}, *AC, CtxI, DT); if (DerefRK) { DerefBytesSCEV = SE.getUMaxExpr( DerefBytesSCEV, SE.getConstant(WiderTy, DerefRK.ArgValue)); @@ -260,12 +263,12 @@ evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, SE.getMinusSCEV(AR->getStart(), StartPtr), WiderTy); const SCEV *OffsetAtLastIter = - mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE); + mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE, CtxI); if (!OffsetAtLastIter) return false; const SCEV *OffsetEndBytes = addSCEVNoOverflow( - OffsetAtLastIter, SE.getNoopOrZeroExtend(EltSize, WiderTy), SE); + OffsetAtLastIter, SE.getNoopOrZeroExtend(EltSize, WiderTy), SE, CtxI); if (!OffsetEndBytes) return false; @@ -273,7 +276,8 @@ evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, // For positive steps, check if // (AR->getStart() - StartPtr) + (MaxBTC * Step) + EltSize <= DerefBytes, // while making sure none of the computations unsigned wrap themselves. - const SCEV *EndBytes = addSCEVNoOverflow(StartOffset, OffsetEndBytes, SE); + const SCEV *EndBytes = + addSCEVNoOverflow(StartOffset, OffsetEndBytes, SE, CtxI); if (!EndBytes) return false; return SE.isKnownPredicate(CmpInst::ICMP_ULE, EndBytes, DerefBytesSCEV); diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 8b7d0e338e5ef..081803c59042f 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -2338,15 +2338,23 @@ bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed, // Can we use context to prove the fact we need? if (!CtxI) return false; - // TODO: Support mul. - if (BinOp == Instruction::Mul) - return false; auto *RHSC = dyn_cast(RHS); // TODO: Lift this limitation. if (!RHSC) return false; APInt C = RHSC->getAPInt(); unsigned NumBits = C.getBitWidth(); + if (BinOp == Instruction::Mul) { + // Multiplying by 0 or 1 never overflows + if (C.isZero() || C.isOne()) + return true; + if (Signed) + return false; + APInt Limit = APInt::getMaxValue(NumBits).udiv(C); + // To avoid overflow, we need to make sure that LHS <= MAX / C. + return isKnownPredicateAt(ICmpInst::ICMP_ULE, LHS, getConstant(Limit), + CtxI); + } bool IsSub = (BinOp == Instruction::Sub); bool IsNegativeConst = (Signed && C.isNegative()); // Compute the direction and magnitude by which we need to check overflow. diff --git a/llvm/test/Transforms/LoopVectorize/vect.stats.ll b/llvm/test/Transforms/LoopVectorize/vect.stats.ll index 018e2c213ddf2..ccac39da2d81e 100644 --- a/llvm/test/Transforms/LoopVectorize/vect.stats.ll +++ b/llvm/test/Transforms/LoopVectorize/vect.stats.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize -enable-early-exit-vectorization --disable-output -stats -S 2>&1 | FileCheck %s +; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize -enable-early-exit-vectorization -use-symbolic-maxbtc-deref-loop --disable-output -stats -S 2>&1 | FileCheck %s ; REQUIRES: asserts ; We have 3 loops, two of them are vectorizable (with one being early-exit From 6cd6e8ba2f4d8e30c358172fb05faf25e377d52c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 3 Sep 2025 10:46:46 +0100 Subject: [PATCH 21/60] [LV] Add additional tests for reasoning about dereferenceable loads. Includes a test for the crash exposed by 08001cf340185877. (cherry picked from commit b16930204b230240d834f667c8f32b12ca4ad198) --- .../single-early-exit-deref-assumptions.ll | 58 ++++- .../LoopVectorize/single_early_exit.ll | 238 ++++++++++++++++++ 2 files changed, 294 insertions(+), 2 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index f329a18f3eaea..c4d0278563b75 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -120,8 +120,62 @@ loop.end: ret i64 %retval } -define i64 @early_exit_alignment_and_deref_known_via_assumption(ptr noalias %p1, ptr noalias %p2, i64 %n) nofree nosync { -; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption( +define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero(ptr noalias %p1, ptr noalias %p2, i64 %n) nofree nosync { +; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero( +; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 [[N]]) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 [[N]]) ] +; CHECK-NEXT: [[C:%.*]] = icmp ne i64 [[N]], 0 +; CHECK-NEXT: br i1 [[C]], label [[LOOP_PREHEADER:%.*]], label [[LOOP_END:%.*]] +; CHECK: loop.preheader: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]] +; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]] +; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1 +; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] +; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END_LOOPEXIT:%.*]] +; CHECK: loop.inc: +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END_LOOPEXIT]] +; CHECK: loop.end.loopexit: +; CHECK-NEXT: [[RETVAL_PH:%.*]] = phi i64 [ -1, [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ] +; CHECK-NEXT: br label [[LOOP_END]] +; CHECK: loop.end: +; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[RETVAL_PH]], [[LOOP_END_LOOPEXIT]] ] +; CHECK-NEXT: ret i64 [[RETVAL]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 %n) ] + call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 %n) ] + %c = icmp ne i64 %n, 0 + br i1 %c, label %loop, label %loop.end + +loop: + %index = phi i64 [ %index.next, %loop.inc ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index + %ld1 = load i8, ptr %arrayidx, align 1 + %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index + %ld2 = load i8, ptr %arrayidx1, align 1 + %cmp3 = icmp eq i8 %ld1, %ld2 + br i1 %cmp3, label %loop.inc, label %loop.end + +loop.inc: + %index.next = add i64 %index, 1 + %exitcond = icmp ne i64 %index.next, %n + br i1 %exitcond, label %loop, label %loop.end + +loop.end: + %retval = phi i64 [ 0, %entry ], [ %index, %loop ], [ -1, %loop.inc ] + ret i64 %retval +} + +define i64 @early_exit_alignment_and_deref_known_via_assumption_n_may_be_zero(ptr noalias %p1, ptr noalias %p2, i64 %n) nofree nosync { +; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_n_may_be_zero( ; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: entry: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 [[N]]) ] diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll index 2069570d4a10f..82d0609dc75dc 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll @@ -334,6 +334,244 @@ early.exit: for.end: ret i32 0 } + +define void @inner_loop_trip_count_depends_on_outer_iv(ptr align 8 dereferenceable(1792) %this, ptr %dst) { +; CHECK-LABEL: define void @inner_loop_trip_count_depends_on_outer_iv( +; CHECK-SAME: ptr align 8 dereferenceable(1792) [[THIS:%.*]], ptr [[DST:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr i8, ptr [[THIS]], i64 1000 +; CHECK-NEXT: br label [[OUTER_HEADER:%.*]] +; CHECK: outer.header: +; CHECK-NEXT: [[OUTER_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[OUTER_IV_NEXT:%.*]], [[OUTER_LATCH:%.*]] ] +; CHECK-NEXT: [[C_1:%.*]] = icmp eq i64 [[OUTER_IV]], 0 +; CHECK-NEXT: br i1 [[C_1]], label [[THEN:%.*]], label [[INNER_HEADER_PREHEADER:%.*]] +; CHECK: inner.header.preheader: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[OUTER_IV]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[OUTER_IV]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[OUTER_IV]], [[N_MOD_VF]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[GEP_SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x ptr>, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x ptr> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] +; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP3]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[OUTER_IV]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[OUTER_LATCH_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: br label [[THEN_LOOPEXIT:%.*]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[INNER_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label [[INNER_HEADER:%.*]] +; CHECK: inner.header: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[INNER_LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[GEP_IV:%.*]] = getelementptr ptr, ptr [[GEP_SRC]], i64 [[IV]] +; CHECK-NEXT: [[L:%.*]] = load ptr, ptr [[GEP_IV]], align 8 +; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[L]], null +; CHECK-NEXT: br i1 [[C_2]], label [[THEN_LOOPEXIT]], label [[INNER_LATCH]] +; CHECK: inner.latch: +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[OUTER_IV]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[OUTER_LATCH_LOOPEXIT]], label [[INNER_HEADER]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: then.loopexit: +; CHECK-NEXT: br label [[THEN]] +; CHECK: then: +; CHECK-NEXT: store i32 0, ptr [[DST]], align 4 +; CHECK-NEXT: br label [[OUTER_LATCH]] +; CHECK: outer.latch.loopexit: +; CHECK-NEXT: br label [[OUTER_LATCH]] +; CHECK: outer.latch: +; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1 +; CHECK-NEXT: [[OUTER_EC:%.*]] = icmp eq i64 [[OUTER_IV_NEXT]], 100 +; CHECK-NEXT: br i1 [[OUTER_EC]], label [[EXIT:%.*]], label [[OUTER_HEADER]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %gep.src = getelementptr i8, ptr %this, i64 1000 + br label %outer.header + +outer.header: + %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ] + %c.1 = icmp eq i64 %outer.iv, 0 + br i1 %c.1, label %then, label %inner.header + +inner.header: + %iv = phi i64 [ %iv.next, %inner.latch ], [ 0, %outer.header ] + %gep.iv = getelementptr ptr, ptr %gep.src, i64 %iv + %l = load ptr, ptr %gep.iv, align 8 + %c.2 = icmp eq ptr %l, null + br i1 %c.2, label %then, label %inner.latch + +inner.latch: + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %outer.iv + br i1 %ec, label %outer.latch, label %inner.header + +then: + store i32 0, ptr %dst, align 4 + br label %outer.latch + +outer.latch: + %outer.iv.next = add i64 %outer.iv, 1 + %outer.ec = icmp eq i64 %outer.iv.next, 100 + br i1 %outer.ec, label %exit, label %outer.header + +exit: + ret void +} + +define i64 @loop_guard_needed_to_prove_dereferenceable(i32 %x, i1 %cmp2) { +; CHECK-LABEL: define i64 @loop_guard_needed_to_prove_dereferenceable( +; CHECK-SAME: i32 [[X:%.*]], i1 [[CMP2:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[A:%.*]] = alloca [32 x i32], align 4 +; CHECK-NEXT: call void @init_mem(ptr [[A]], i64 128) +; CHECK-NEXT: [[C_X:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: br i1 [[C_X]], label [[PH:%.*]], label [[EXIT:%.*]] +; CHECK: ph: +; CHECK-NEXT: [[N:%.*]] = tail call i32 @llvm.smin.i32(i32 [[X]], i32 31) +; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = add nuw nsw i64 [[N_EXT]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP0]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [32 x i32], ptr [[A]], i64 0, i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] +; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP0]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 true) +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], [[TMP7]] +; CHECK-NEXT: br label [[EXIT_LOOPEXIT]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[PH]] ] +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] +; CHECK-NEXT: [[ARRAYIDX42:%.*]] = getelementptr [32 x i32], ptr [[A]], i64 0, i64 [[IV]] +; CHECK-NEXT: [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX42]], align 4 +; CHECK-NEXT: [[CMP43:%.*]] = icmp eq i32 [[TMP9]], 0 +; CHECK-NEXT: br i1 [[CMP43]], label [[EXIT_LOOPEXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N_EXT]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK: exit.loopexit: +; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV]], [[LOOP_HEADER]] ], [ -1, [[LOOP_LATCH]] ], [ -1, [[MIDDLE_BLOCK]] ], [ [[TMP8]], [[VECTOR_EARLY_EXIT]] ] +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: [[RES:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[RES_PH]], [[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret i64 [[RES]] +; +entry: + %A = alloca [32 x i32], align 4 + call void @init_mem(ptr %A, i64 128) + %c.x = icmp sgt i32 %x, 0 + br i1 %c.x, label %ph, label %exit + +ph: + %n = tail call i32 @llvm.smin.i32(i32 %x, i32 31) + %n.ext = zext i32 %n to i64 + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop.latch ] + %arrayidx42 = getelementptr [32 x i32], ptr %A, i64 0, i64 %iv + %0 = load i32, ptr %arrayidx42, align 4 + %cmp43 = icmp eq i32 %0, 0 + br i1 %cmp43, label %exit, label %loop.latch + +loop.latch: + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, %n.ext + br i1 %ec, label %exit, label %loop.header + +exit: + %res = phi i64 [ 0, %entry ], [ -1, %loop.latch ], [ %iv, %loop.header ] + ret i64 %res +} + +declare i32 @llvm.smin.i32(i32, i32) + +@A = external global [100 x {i32, i8} ] + +define ptr @btc_and_max_btc_require_predicates(ptr noalias %start, i64 %offset) { +; CHECK-LABEL: define ptr @btc_and_max_btc_require_predicates( +; CHECK-SAME: ptr noalias [[START:%.*]], i64 [[OFFSET:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[END:%.*]] = getelementptr i32, ptr [[START]], i64 [[OFFSET]] +; CHECK-NEXT: [[PRE_1:%.*]] = icmp ult i64 [[OFFSET]], 100 +; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_1]]) +; CHECK-NEXT: [[PRE_2:%.*]] = icmp ugt i64 [[OFFSET]], 1 +; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_2]]) +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: [[IV_1:%.*]] = phi ptr [ @A, [[ENTRY:%.*]] ], [ [[IV_1_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] +; CHECK-NEXT: [[IV_2:%.*]] = phi ptr [ [[START]], [[ENTRY]] ], [ [[IV_2_NEXT:%.*]], [[LOOP_LATCH]] ] +; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[IV_1]], align 4 +; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 0 +; CHECK-NEXT: br i1 [[C]], label [[LOOP_LATCH]], label [[EXIT:%.*]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_2_NEXT]] = getelementptr i8, ptr [[IV_2]], i64 40 +; CHECK-NEXT: [[IV_1_NEXT]] = getelementptr i8, ptr [[IV_1]], i64 40 +; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[IV_2]], [[END]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT]], label [[LOOP_HEADER]] +; CHECK: exit: +; CHECK-NEXT: [[RES:%.*]] = phi ptr [ [[IV_1]], [[LOOP_HEADER]] ], [ [[IV_2]], [[LOOP_LATCH]] ] +; CHECK-NEXT: ret ptr [[RES]] +; +entry: + %end = getelementptr i32, ptr %start, i64 %offset + %pre.1 = icmp ult i64 %offset, 100 + call void @llvm.assume(i1 %pre.1) + %pre.2 = icmp ugt i64 %offset, 1 + call void @llvm.assume(i1 %pre.2) + br label %loop.header + +loop.header: + %iv.1 = phi ptr [ @A, %entry ], [ %iv.1.next, %loop.latch ] + %iv.2 = phi ptr [ %start, %entry ], [ %iv.2.next, %loop.latch ] + %l = load i32, ptr %iv.1, align 4 + %c = icmp eq i32 %l, 0 + br i1 %c, label %loop.latch, label %exit + +loop.latch: + %iv.2.next = getelementptr i8, ptr %iv.2, i64 40 + %iv.1.next = getelementptr i8, ptr %iv.1, i64 40 + %ec = icmp eq ptr %iv.2, %end + br i1 %ec, label %exit, label %loop.header + +exit: + %res = phi ptr [ %iv.1, %loop.header ], [ %iv.2, %loop.latch ] + ret ptr %res +} + ;. ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} From 57bd0c8e0b80dd213f0868e834107ddb1825c50e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 3 Sep 2025 12:45:28 +0100 Subject: [PATCH 22/60] Reapply "[LAA,Loads] Use loop guards and max BTC if needed when checking deref. (#155672)" This reverts commit f0df1e3dd4ec064821f673ced7d83e5a2cf6afa1. Recommit with extra check for SCEVCouldNotCompute. Test has been added in b16930204b. Original message: Remove the fall-back to constant max BTC if the backedge-taken-count cannot be computed. The constant max backedge-taken count is computed considering loop guards, so to avoid regressions we need to apply loop guards as needed. Also remove the special handling for Mul in willNotOverflow, as this should not longer be needed after 914374624f (https://github.com/llvm/llvm-project/pull/155300). PR: https://github.com/llvm/llvm-project/pull/155672 (cherry picked from commit a434a7a4f12faf3e0231fc145f7fb2e02c623a97) --- .../llvm/Analysis/LoopAccessAnalysis.h | 22 ++++-- llvm/lib/Analysis/Loads.cpp | 35 ++++------ llvm/lib/Analysis/LoopAccessAnalysis.cpp | 69 +++++++++++-------- llvm/lib/Analysis/ScalarEvolution.cpp | 14 +--- .../Transforms/LoopVectorize/vect.stats.ll | 2 +- 5 files changed, 77 insertions(+), 65 deletions(-) diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 92304edd67a44..52ab38583d5de 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -183,10 +183,12 @@ class MemoryDepChecker { MemoryDepChecker(PredicatedScalarEvolution &PSE, AssumptionCache *AC, DominatorTree *DT, const Loop *L, const DenseMap &SymbolicStrides, - unsigned MaxTargetVectorWidthInBits) + unsigned MaxTargetVectorWidthInBits, + std::optional &LoopGuards) : PSE(PSE), AC(AC), DT(DT), InnermostLoop(L), SymbolicStrides(SymbolicStrides), - MaxTargetVectorWidthInBits(MaxTargetVectorWidthInBits) {} + MaxTargetVectorWidthInBits(MaxTargetVectorWidthInBits), + LoopGuards(LoopGuards) {} /// Register the location (instructions are given increasing numbers) /// of a write access. @@ -373,7 +375,7 @@ class MemoryDepChecker { PointerBounds; /// Cache for the loop guards of InnermostLoop. - std::optional LoopGuards; + std::optional &LoopGuards; /// Check whether there is a plausible dependence between the two /// accesses. @@ -531,8 +533,9 @@ class RuntimePointerChecking { AliasSetId(AliasSetId), Expr(Expr), NeedsFreeze(NeedsFreeze) {} }; - RuntimePointerChecking(MemoryDepChecker &DC, ScalarEvolution *SE) - : DC(DC), SE(SE) {} + RuntimePointerChecking(MemoryDepChecker &DC, ScalarEvolution *SE, + std::optional &LoopGuards) + : DC(DC), SE(SE), LoopGuards(LoopGuards) {} /// Reset the state of the pointer runtime information. void reset() { @@ -646,6 +649,9 @@ class RuntimePointerChecking { /// Holds a pointer to the ScalarEvolution analysis. ScalarEvolution *SE; + /// Cache for the loop guards of the loop. + std::optional &LoopGuards; + /// Set of run-time checks required to establish independence of /// otherwise may-aliasing pointers in the loop. SmallVector Checks; @@ -821,6 +827,9 @@ class LoopAccessInfo { Loop *TheLoop; + /// Cache for the loop guards of TheLoop. + std::optional LoopGuards; + /// Determines whether we should generate partial runtime checks when not all /// memory accesses could be analyzed. bool AllowPartial; @@ -938,7 +947,8 @@ LLVM_ABI std::pair getStartAndEndForAccess( const SCEV *MaxBTC, ScalarEvolution *SE, DenseMap, std::pair> *PointerBounds, - DominatorTree *DT, AssumptionCache *AC); + DominatorTree *DT, AssumptionCache *AC, + std::optional &LoopGuards); class LoopAccessInfoManager { /// The cache. diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index 94af3cc5861c3..9c6b77b775b9d 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -26,10 +26,6 @@ using namespace llvm; -static cl::opt - UseSymbolicMaxBTCForDerefInLoop("use-symbolic-maxbtc-deref-loop", - cl::init(false)); - static bool isAligned(const Value *Base, Align Alignment, const DataLayout &DL) { return Base->getPointerAlignment(DL) >= Alignment; @@ -336,18 +332,10 @@ bool llvm::isDereferenceableAndAlignedInLoop( : SE.getBackedgeTakenCount(L); if (isa(MaxBECount)) return false; - - if (isa(BECount) && !UseSymbolicMaxBTCForDerefInLoop) { - // TODO: Support symbolic max backedge taken counts for loops without - // computable backedge taken counts. - MaxBECount = - Predicates - ? SE.getPredicatedConstantMaxBackedgeTakenCount(L, *Predicates) - : SE.getConstantMaxBackedgeTakenCount(L); - } - - const auto &[AccessStart, AccessEnd] = getStartAndEndForAccess( - L, PtrScev, LI->getType(), BECount, MaxBECount, &SE, nullptr, &DT, AC); + std::optional LoopGuards; + const auto &[AccessStart, AccessEnd] = + getStartAndEndForAccess(L, PtrScev, LI->getType(), BECount, MaxBECount, + &SE, nullptr, &DT, AC, LoopGuards); if (isa(AccessStart) || isa(AccessEnd)) return false; @@ -356,7 +344,13 @@ bool llvm::isDereferenceableAndAlignedInLoop( const SCEV *PtrDiff = SE.getMinusSCEV(AccessEnd, AccessStart); if (isa(PtrDiff)) return false; - APInt MaxPtrDiff = SE.getUnsignedRangeMax(PtrDiff); + + if (!LoopGuards) + LoopGuards.emplace( + ScalarEvolution::LoopGuards::collect(AddRec->getLoop(), SE)); + + APInt MaxPtrDiff = + SE.getUnsignedRangeMax(SE.applyLoopGuards(PtrDiff, *LoopGuards)); Value *Base = nullptr; APInt AccessSize; @@ -399,9 +393,10 @@ bool llvm::isDereferenceableAndAlignedInLoop( Instruction *HeaderFirstNonPHI = &*L->getHeader()->getFirstNonPHIIt(); return isDereferenceableAndAlignedPointerViaAssumption( Base, Alignment, - [&SE, AccessSizeSCEV](const RetainedKnowledge &RK) { - return SE.isKnownPredicate(CmpInst::ICMP_ULE, AccessSizeSCEV, - SE.getSCEV(RK.IRArgValue)); + [&SE, AccessSizeSCEV, &LoopGuards](const RetainedKnowledge &RK) { + return SE.isKnownPredicate( + CmpInst::ICMP_ULE, AccessSizeSCEV, + SE.applyLoopGuards(SE.getSCEV(RK.IRArgValue), *LoopGuards)); }, DL, HeaderFirstNonPHI, AC, &DT) || isDereferenceableAndAlignedPointer(Base, Alignment, AccessSize, DL, diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index cf4c5785f1e26..1e60d8f654c1e 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -193,9 +193,8 @@ RuntimeCheckingPtrGroup::RuntimeCheckingPtrGroup( /// Returns \p A + \p B, if it is guaranteed not to unsigned wrap. Otherwise /// return nullptr. \p A and \p B must have the same type. static const SCEV *addSCEVNoOverflow(const SCEV *A, const SCEV *B, - ScalarEvolution &SE, - const Instruction *CtxI) { - if (!SE.willNotOverflow(Instruction::Add, /*IsSigned=*/false, A, B, CtxI)) + ScalarEvolution &SE) { + if (!SE.willNotOverflow(Instruction::Add, /*IsSigned=*/false, A, B)) return nullptr; return SE.getAddExpr(A, B); } @@ -203,20 +202,19 @@ static const SCEV *addSCEVNoOverflow(const SCEV *A, const SCEV *B, /// Returns \p A * \p B, if it is guaranteed not to unsigned wrap. Otherwise /// return nullptr. \p A and \p B must have the same type. static const SCEV *mulSCEVOverflow(const SCEV *A, const SCEV *B, - ScalarEvolution &SE, - const Instruction *CtxI) { - if (!SE.willNotOverflow(Instruction::Mul, /*IsSigned=*/false, A, B, CtxI)) + ScalarEvolution &SE) { + if (!SE.willNotOverflow(Instruction::Mul, /*IsSigned=*/false, A, B)) return nullptr; return SE.getMulExpr(A, B); } /// Return true, if evaluating \p AR at \p MaxBTC cannot wrap, because \p AR at /// \p MaxBTC is guaranteed inbounds of the accessed object. -static bool -evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, - const SCEV *MaxBTC, const SCEV *EltSize, - ScalarEvolution &SE, const DataLayout &DL, - DominatorTree *DT, AssumptionCache *AC) { +static bool evaluatePtrAddRecAtMaxBTCWillNotWrap( + const SCEVAddRecExpr *AR, const SCEV *MaxBTC, const SCEV *EltSize, + ScalarEvolution &SE, const DataLayout &DL, DominatorTree *DT, + AssumptionCache *AC, + std::optional &LoopGuards) { auto *PointerBase = SE.getPointerBase(AR->getStart()); auto *StartPtr = dyn_cast(PointerBase); if (!StartPtr) @@ -234,12 +232,11 @@ evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, Type *WiderTy = SE.getWiderType(MaxBTC->getType(), Step->getType()); const SCEV *DerefBytesSCEV = SE.getConstant(WiderTy, DerefBytes); - // Context which dominates the entire loop. - auto *CtxI = L->getLoopPredecessor()->getTerminator(); // Check if we have a suitable dereferencable assumption we can use. if (!StartPtrV->canBeFreed()) { RetainedKnowledge DerefRK = getKnowledgeValidInContext( - StartPtrV, {Attribute::Dereferenceable}, *AC, CtxI, DT); + StartPtrV, {Attribute::Dereferenceable}, *AC, + L->getLoopPredecessor()->getTerminator(), DT); if (DerefRK) { DerefBytesSCEV = SE.getUMaxExpr( DerefBytesSCEV, SE.getConstant(WiderTy, DerefRK.ArgValue)); @@ -263,12 +260,23 @@ evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, SE.getMinusSCEV(AR->getStart(), StartPtr), WiderTy); const SCEV *OffsetAtLastIter = - mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE, CtxI); - if (!OffsetAtLastIter) - return false; + mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE); + if (!OffsetAtLastIter) { + // Re-try with constant max backedge-taken count if using the symbolic one + // failed. + MaxBTC = SE.getConstantMaxBackedgeTakenCount(AR->getLoop()); + if (isa(MaxBTC)) + return false; + MaxBTC = SE.getNoopOrZeroExtend( + MaxBTC, WiderTy); + OffsetAtLastIter = + mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE); + if (!OffsetAtLastIter) + return false; + } const SCEV *OffsetEndBytes = addSCEVNoOverflow( - OffsetAtLastIter, SE.getNoopOrZeroExtend(EltSize, WiderTy), SE, CtxI); + OffsetAtLastIter, SE.getNoopOrZeroExtend(EltSize, WiderTy), SE); if (!OffsetEndBytes) return false; @@ -276,10 +284,15 @@ evaluatePtrAddRecAtMaxBTCWillNotWrap(const SCEVAddRecExpr *AR, // For positive steps, check if // (AR->getStart() - StartPtr) + (MaxBTC * Step) + EltSize <= DerefBytes, // while making sure none of the computations unsigned wrap themselves. - const SCEV *EndBytes = - addSCEVNoOverflow(StartOffset, OffsetEndBytes, SE, CtxI); + const SCEV *EndBytes = addSCEVNoOverflow(StartOffset, OffsetEndBytes, SE); if (!EndBytes) return false; + + if (!LoopGuards) + LoopGuards.emplace( + ScalarEvolution::LoopGuards::collect(AR->getLoop(), SE)); + + EndBytes = SE.applyLoopGuards(EndBytes, *LoopGuards); return SE.isKnownPredicate(CmpInst::ICMP_ULE, EndBytes, DerefBytesSCEV); } @@ -296,7 +309,8 @@ std::pair llvm::getStartAndEndForAccess( const SCEV *MaxBTC, ScalarEvolution *SE, DenseMap, std::pair> *PointerBounds, - DominatorTree *DT, AssumptionCache *AC) { + DominatorTree *DT, AssumptionCache *AC, + std::optional &LoopGuards) { std::pair *PtrBoundsPair; if (PointerBounds) { auto [Iter, Ins] = PointerBounds->insert( @@ -332,7 +346,7 @@ std::pair llvm::getStartAndEndForAccess( // separately checks that accesses cannot not wrap, so unsigned max // represents an upper bound. if (evaluatePtrAddRecAtMaxBTCWillNotWrap(AR, MaxBTC, EltSizeSCEV, *SE, DL, - DT, AC)) { + DT, AC, LoopGuards)) { ScEnd = AR->evaluateAtIteration(MaxBTC, *SE); } else { ScEnd = SE->getAddExpr( @@ -381,7 +395,7 @@ void RuntimePointerChecking::insert(Loop *Lp, Value *Ptr, const SCEV *PtrExpr, const SCEV *BTC = PSE.getBackedgeTakenCount(); const auto &[ScStart, ScEnd] = getStartAndEndForAccess( Lp, PtrExpr, AccessTy, BTC, SymbolicMaxBTC, PSE.getSE(), - &DC.getPointerBounds(), DC.getDT(), DC.getAC()); + &DC.getPointerBounds(), DC.getDT(), DC.getAC(), LoopGuards); assert(!isa(ScStart) && !isa(ScEnd) && "must be able to compute both start and end expressions"); @@ -1986,13 +2000,13 @@ bool MemoryDepChecker::areAccessesCompletelyBeforeOrAfter(const SCEV *Src, ScalarEvolution &SE = *PSE.getSE(); const auto &[SrcStart_, SrcEnd_] = getStartAndEndForAccess(InnermostLoop, Src, SrcTy, BTC, SymbolicMaxBTC, - &SE, &PointerBounds, DT, AC); + &SE, &PointerBounds, DT, AC, LoopGuards); if (isa(SrcStart_) || isa(SrcEnd_)) return false; const auto &[SinkStart_, SinkEnd_] = getStartAndEndForAccess(InnermostLoop, Sink, SinkTy, BTC, SymbolicMaxBTC, - &SE, &PointerBounds, DT, AC); + &SE, &PointerBounds, DT, AC, LoopGuards); if (isa(SinkStart_) || isa(SinkEnd_)) return false; @@ -3039,8 +3053,9 @@ LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE, TTI->getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector) * 2; DepChecker = std::make_unique( - *PSE, AC, DT, L, SymbolicStrides, MaxTargetVectorWidthInBits); - PtrRtChecking = std::make_unique(*DepChecker, SE); + *PSE, AC, DT, L, SymbolicStrides, MaxTargetVectorWidthInBits, LoopGuards); + PtrRtChecking = + std::make_unique(*DepChecker, SE, LoopGuards); if (canAnalyzeLoop()) CanVecMem = analyzeLoop(AA, LI, TLI, DT); } diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 081803c59042f..8b7d0e338e5ef 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -2338,23 +2338,15 @@ bool ScalarEvolution::willNotOverflow(Instruction::BinaryOps BinOp, bool Signed, // Can we use context to prove the fact we need? if (!CtxI) return false; + // TODO: Support mul. + if (BinOp == Instruction::Mul) + return false; auto *RHSC = dyn_cast(RHS); // TODO: Lift this limitation. if (!RHSC) return false; APInt C = RHSC->getAPInt(); unsigned NumBits = C.getBitWidth(); - if (BinOp == Instruction::Mul) { - // Multiplying by 0 or 1 never overflows - if (C.isZero() || C.isOne()) - return true; - if (Signed) - return false; - APInt Limit = APInt::getMaxValue(NumBits).udiv(C); - // To avoid overflow, we need to make sure that LHS <= MAX / C. - return isKnownPredicateAt(ICmpInst::ICMP_ULE, LHS, getConstant(Limit), - CtxI); - } bool IsSub = (BinOp == Instruction::Sub); bool IsNegativeConst = (Signed && C.isNegative()); // Compute the direction and magnitude by which we need to check overflow. diff --git a/llvm/test/Transforms/LoopVectorize/vect.stats.ll b/llvm/test/Transforms/LoopVectorize/vect.stats.ll index ccac39da2d81e..018e2c213ddf2 100644 --- a/llvm/test/Transforms/LoopVectorize/vect.stats.ll +++ b/llvm/test/Transforms/LoopVectorize/vect.stats.ll @@ -1,4 +1,4 @@ -; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize -enable-early-exit-vectorization -use-symbolic-maxbtc-deref-loop --disable-output -stats -S 2>&1 | FileCheck %s +; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=4 -force-vector-width=4 -debug-only=loop-vectorize -enable-early-exit-vectorization --disable-output -stats -S 2>&1 | FileCheck %s ; REQUIRES: asserts ; We have 3 loops, two of them are vectorizable (with one being early-exit From 29e4e6a9be9032699cefdfae3526e65c7aec32d9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 4 Sep 2025 11:32:33 +0100 Subject: [PATCH 23/60] [LAA] Support assumptions with non-constant deref sizes. (#156758) Update evaluatePtrAddrecAtMaxBTCWillNotWrap to support non-constant sizes in dereferenceable assumptions. Apply loop-guards in a few places needed to reason about expressions involving trip counts of the from (BTC - 1). PR: https://github.com/llvm/llvm-project/pull/156758 (cherry picked from commit b400fd115145ccea0b62944e37e74eedc9da223f) --- llvm/lib/Analysis/Loads.cpp | 3 +- llvm/lib/Analysis/LoopAccessAnalysis.cpp | 14 +- .../single-early-exit-deref-assumptions.ll | 163 +++++++++++++++++- .../LoopVectorize/single_early_exit.ll | 16 +- 4 files changed, 174 insertions(+), 22 deletions(-) diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index 9c6b77b775b9d..0d708f1093c28 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -395,7 +395,8 @@ bool llvm::isDereferenceableAndAlignedInLoop( Base, Alignment, [&SE, AccessSizeSCEV, &LoopGuards](const RetainedKnowledge &RK) { return SE.isKnownPredicate( - CmpInst::ICMP_ULE, AccessSizeSCEV, + CmpInst::ICMP_ULE, + SE.applyLoopGuards(AccessSizeSCEV, *LoopGuards), SE.applyLoopGuards(SE.getSCEV(RK.IRArgValue), *LoopGuards)); }, DL, HeaderFirstNonPHI, AC, &DT) || diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 1e60d8f654c1e..1b8ef1b96ad94 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -238,8 +238,8 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap( StartPtrV, {Attribute::Dereferenceable}, *AC, L->getLoopPredecessor()->getTerminator(), DT); if (DerefRK) { - DerefBytesSCEV = SE.getUMaxExpr( - DerefBytesSCEV, SE.getConstant(WiderTy, DerefRK.ArgValue)); + DerefBytesSCEV = + SE.getUMaxExpr(DerefBytesSCEV, SE.getSCEV(DerefRK.IRArgValue)); } } @@ -259,6 +259,10 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap( const SCEV *StartOffset = SE.getNoopOrZeroExtend( SE.getMinusSCEV(AR->getStart(), StartPtr), WiderTy); + if (!LoopGuards) + LoopGuards.emplace(ScalarEvolution::LoopGuards::collect(AR->getLoop(), SE)); + MaxBTC = SE.applyLoopGuards(MaxBTC, *LoopGuards); + const SCEV *OffsetAtLastIter = mulSCEVOverflow(MaxBTC, SE.getAbsExpr(Step, /*IsNSW=*/false), SE); if (!OffsetAtLastIter) { @@ -288,11 +292,7 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap( if (!EndBytes) return false; - if (!LoopGuards) - LoopGuards.emplace( - ScalarEvolution::LoopGuards::collect(AR->getLoop(), SE)); - - EndBytes = SE.applyLoopGuards(EndBytes, *LoopGuards); + DerefBytesSCEV = SE.applyLoopGuards(DerefBytesSCEV, *LoopGuards); return SE.isKnownPredicate(CmpInst::ICMP_ULE, EndBytes, DerefBytesSCEV); } diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index c4d0278563b75..a591423e6fc91 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -13,9 +13,11 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_si ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX1]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP0]], align 1 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX1]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP2]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) @@ -25,13 +27,13 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_si ; CHECK: middle.split: ; CHECK-NEXT: br i1 [[TMP5]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] ; CHECK: middle.block: -; CHECK-NEXT: br label [[LOOP_END:%.*]] +; CHECK-NEXT: br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]] ; CHECK: vector.early.exit: ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true) ; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX1]], [[TMP8]] ; CHECK-NEXT: br label [[LOOP_END]] ; CHECK: scalar.ph: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: br label [[LOOP1:%.*]] ; CHECK: loop: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] @@ -129,21 +131,52 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero(ptr n ; CHECK-NEXT: [[C:%.*]] = icmp ne i64 [[N]], 0 ; CHECK-NEXT: br i1 [[C]], label [[LOOP_PREHEADER:%.*]], label [[LOOP_END:%.*]] ; CHECK: loop.preheader: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX1]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX1]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] +; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]] +; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] +; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOP_END_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 true) +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], [[TMP7]] +; CHECK-NEXT: br label [[LOOP_END_LOOPEXIT]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[LOOP_PREHEADER]] ] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]] ; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 ; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]] ; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1 ; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] -; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END_LOOPEXIT:%.*]] +; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END_LOOPEXIT]] ; CHECK: loop.inc: ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1 ; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END_LOOPEXIT]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END_LOOPEXIT]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: loop.end.loopexit: -; CHECK-NEXT: [[RETVAL_PH:%.*]] = phi i64 [ -1, [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ] +; CHECK-NEXT: [[RETVAL_PH:%.*]] = phi i64 [ -1, [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ], [ -1, [[MIDDLE_BLOCK]] ], [ [[TMP8]], [[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: br label [[LOOP_END]] ; CHECK: loop.end: ; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[RETVAL_PH]], [[LOOP_END_LOOPEXIT]] ] @@ -220,3 +253,117 @@ loop.end: %retval = phi i64 [ %index, %loop ], [ -1, %loop.inc ] ret i64 %retval } + +define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero_i16(ptr noalias %p1, ptr noalias %p2, i32 %n) nofree nosync { +; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero_i16( +; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]], i32 [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: [[N_SCALED:%.*]] = shl nuw nsw i64 [[N_EXT]], 1 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 [[N_SCALED]]) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 [[N_SCALED]]) ] +; CHECK-NEXT: [[C:%.*]] = icmp ne i64 [[N_EXT]], 0 +; CHECK-NEXT: br i1 [[C]], label [[LOOP_PREHEADER:%.*]], label [[LOOP_END:%.*]] +; CHECK: loop.preheader: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]] +; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]] +; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1 +; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] +; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END_LOOPEXIT:%.*]] +; CHECK: loop.inc: +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], [[N_EXT]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END_LOOPEXIT]] +; CHECK: loop.end.loopexit: +; CHECK-NEXT: [[RETVAL_PH:%.*]] = phi i64 [ -1, [[LOOP_INC]] ], [ [[INDEX]], [[LOOP]] ] +; CHECK-NEXT: br label [[LOOP_END]] +; CHECK: loop.end: +; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[RETVAL_PH]], [[LOOP_END_LOOPEXIT]] ] +; CHECK-NEXT: ret i64 [[RETVAL]] +; +entry: + %n.ext = zext i32 %n to i64 + %n.scaled = shl nuw nsw i64 %n.ext, 1 + call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 %n.scaled) ] + call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 %n.scaled) ] + %c = icmp ne i64 %n.ext, 0 + br i1 %c, label %loop, label %loop.end + +loop: + %index = phi i64 [ 0, %entry ], [ %index.next, %loop.inc ] + %gep.p1 = getelementptr inbounds i8, ptr %p1, i64 %index + %ld1 = load i8, ptr %gep.p1, align 1 + %gep.p2 = getelementptr inbounds i8, ptr %p2, i64 %index + %ld2 = load i8, ptr %gep.p2, align 1 + %c.0 = icmp eq i8 %ld1, %ld2 + br i1 %c.0, label %loop.inc, label %loop.end + +loop.inc: + %index.next = add i64 %index, 1 + %exitcond = icmp ne i64 %index.next, %n.ext + br i1 %exitcond, label %loop, label %loop.end + +loop.end: + %retval = phi i64 [ 0, %entry ], [ %index, %loop ], [ -1, %loop.inc ] + ret i64 %retval +} + +define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero_i16_ptr_iv(ptr %A, i32 noundef %n) nofree nosync { +; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero_i16_ptr_iv( +; CHECK-SAME: ptr [[A:%.*]], i32 noundef [[N:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 2) ] +; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[N_EXT]], 1 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 [[MUL]]) ] +; CHECK-NEXT: [[A_END:%.*]] = getelementptr i8, ptr [[A]], i64 [[MUL]] +; CHECK-NEXT: [[PRE:%.*]] = icmp eq i32 [[N]], 0 +; CHECK-NEXT: br i1 [[PRE]], label [[EXIT:%.*]], label [[LOOP_HEADER_PREHEADER:%.*]] +; CHECK: loop.header.preheader: +; CHECK-NEXT: br label [[LOOP_HEADER1:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: [[IV1:%.*]] = phi ptr [ [[IV_NEXT1:%.*]], [[LOOP_LATCH1:%.*]] ], [ [[A]], [[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[IV1]], align 2 +; CHECK-NEXT: [[C_0:%.*]] = icmp eq i16 [[L]], 0 +; CHECK-NEXT: br i1 [[C_0]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_LATCH1]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_NEXT1]] = getelementptr inbounds nuw i8, ptr [[IV1]], i64 2 +; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[IV_NEXT1]], [[A_END]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER1]] +; CHECK: exit.loopexit: +; CHECK-NEXT: [[P_PH:%.*]] = phi ptr [ [[A_END]], [[LOOP_LATCH1]] ], [ [[IV1]], [[LOOP_HEADER1]] ] +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[P_PH]], [[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: [[RES:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: ret i64 [[RES]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %A, i64 2) ] + %n.ext = zext i32 %n to i64 + %mul = shl nuw nsw i64 %n.ext, 1 + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 %mul) ] + %A.end = getelementptr i8, ptr %A, i64 %mul + %pre = icmp eq i32 %n, 0 + br i1 %pre, label %exit, label %loop.header + +loop.header: + %iv = phi ptr [ %iv.next, %loop.latch ], [ %A, %entry ] + %l = load i16, ptr %iv, align 2 + %c.0 = icmp eq i16 %l, 0 + br i1 %c.0, label %exit, label %loop.latch + +loop.latch: + %iv.next = getelementptr inbounds nuw i8, ptr %iv, i64 2 + %ec = icmp eq ptr %iv.next, %A.end + br i1 %ec, label %exit, label %loop.header + +exit: + %p = phi ptr [ %A, %entry ], [ %iv, %loop.header ], [ %A.end, %loop.latch ] + %res = ptrtoint ptr %p to i64 + ret i64 %res +} diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll index 82d0609dc75dc..38553b2385b06 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll @@ -355,11 +355,11 @@ define void @inner_loop_trip_count_depends_on_outer_iv(ptr align 8 dereferenceab ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr ptr, ptr [[GEP_SRC]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x ptr>, ptr [[TMP0]], align 8 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr ptr, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x ptr>, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x ptr> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]]) ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -452,11 +452,11 @@ define i64 @loop_guard_needed_to_prove_dereferenceable(i32 %x, i1 %cmp2) { ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr [32 x i32], ptr [[A]], i64 0, i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]] -; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] ; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -583,4 +583,8 @@ exit: ; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]} ; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} ; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} +; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} +; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} +; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} +; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} ;. From b18c2cadf05784be7bf6e634d9a1e112fd6a54fa Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Sep 2025 09:45:13 +0100 Subject: [PATCH 24/60] [SCEV] Fold (C * A /u C) -> A, if A is a multiple of C and C a pow-of-2. (#156730) Alive2 Proof: https://alive2.llvm.org/ce/z/JoHJE9 PR: https://github.com/llvm/llvm-project/pull/156730 (cherry picked from commit 74ec38fad0a1289f936e5388fa8bbe74653c55d9) --- llvm/lib/Analysis/ScalarEvolution.cpp | 9 + ...en-count-guard-info-rewrite-expressions.ll | 72 ++--- .../max-backedge-taken-count-guard-info.ll | 4 +- .../pr58402-large-number-of-zext-exprs.ll | 62 ++-- llvm/test/CodeGen/PowerPC/common-chain.ll | 294 +++++++++--------- .../fixed-vectors-strided-load-store-asm.ll | 7 +- llvm/test/Transforms/LoopIdiom/basic.ll | 5 +- .../X86/zext-signed-addrec.ll | 121 +++++++ .../LoopVectorize/X86/gather_scatter.ll | 42 +-- .../single-early-exit-deref-assumptions.ll | 52 +++- 10 files changed, 414 insertions(+), 254 deletions(-) create mode 100644 llvm/test/Transforms/LoopStrengthReduce/X86/zext-signed-addrec.ll diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 8b7d0e338e5ef..856c13ec9e9e1 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3215,6 +3215,15 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, return getZeroExtendExpr(Res, Ops[1]->getType(), Depth + 1); }; } + + // Try to fold (C * D /u C) -> D, if C is a power-of-2 and D is a multiple + // of C. + const SCEV *D; + if (match(Ops[1], m_scev_UDiv(m_SCEV(D), m_scev_Specific(LHSC))) && + LHSC->getAPInt().isPowerOf2() && + LHSC->getAPInt().logBase2() <= getMinTrailingZeros(D)) { + return D; + } } } diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll index 8c77d704eac6a..4e5033b7a2f7f 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info-rewrite-expressions.ll @@ -12,9 +12,9 @@ define void @rewrite_zext(i32 %n) { ; CHECK-NEXT: %n.vec = and i64 %ext, -8 ; CHECK-NEXT: --> (8 * ((zext i32 %n to i64) /u 8)) U: [0,4294967289) S: [0,4294967289) ; CHECK-NEXT: %index = phi i64 [ 0, %check ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,8}<%loop> U: [0,17) S: [0,17) Exits: (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,8}<%loop> U: [0,17) S: [0,17) Exits: (-8 + (8 * ((zext i32 %n to i64) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw nsw i64 %index, 8 -; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,25) S: [8,25) Exits: (8 + (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,25) S: [8,25) Exits: (8 * ((zext i32 %n to i64) /u 8)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2 @@ -52,11 +52,11 @@ define i32 @rewrite_zext_min_max(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * ((16 umin (zext i32 %N to i64)) /u 4)) U: [0,17) S: [0,17) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * ((16 umin (zext i32 %N to i64)) /u 4)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_min_max ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -98,11 +98,11 @@ define i32 @rewrite_min_max_zext(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %umin, 28 ; CHECK-NEXT: --> (4 * ((16 umin (zext i32 %N to i64)) /u 4)) U: [0,17) S: [0,17) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * ((16 umin (zext i32 %N to i64)) /u 4)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_min_max_zext ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -144,11 +144,11 @@ define i32 @rewrite_sext_min_max(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nsw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_min_max ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -190,11 +190,11 @@ define i32 @rewrite_min_max_sext(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %smin, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nsw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_min_max_sext ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -288,9 +288,9 @@ define i32 @rewrite_zext_no_icmp_ne(i32 %N) { ; CHECK-NEXT: %n.vec = and i64 %n.rnd.up, 8589934588 ; CHECK-NEXT: --> (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4)) U: [4,4294967297) S: [4,4294967297) ; CHECK-NEXT: %iv = phi i64 [ 0, %loop.ph ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,4294967293) S: [0,4294967293) Exits: (4 * ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,4294967293) S: [0,4294967293) Exits: (-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = add i64 %iv, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,4294967297) S: [4,4294967297) Exits: (4 + (4 * ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,4294967297) S: [4,4294967297) Exits: (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_no_icmp_ne ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((4 + (zext i32 (-1 + (zext i2 (trunc i32 %N to i2) to i32)) to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 1073741823 @@ -328,9 +328,9 @@ define void @rewrite_zext_and_base_1(i32 %n) { ; CHECK-NEXT: %n.vec = and i64 %ext, -8 ; CHECK-NEXT: --> (8 * ((zext i32 %n to i64) /u 8)) U: [0,4294967289) S: [0,4294967289) ; CHECK-NEXT: %index = phi i64 [ 0, %check ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,8}<%loop> U: [0,25) S: [0,25) Exits: (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,8}<%loop> U: [0,25) S: [0,25) Exits: (-8 + (8 * ((zext i32 %n to i64) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw nsw i64 %index, 8 -; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,33) S: [8,33) Exits: (8 + (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,33) S: [8,33) Exits: (8 * ((zext i32 %n to i64) /u 8)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_and_base_1 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -371,9 +371,9 @@ define void @rewrite_zext_and_base_2(i32 %n) { ; CHECK-NEXT: %n.vec = and i64 %ext, -8 ; CHECK-NEXT: --> (8 * ((zext i32 %n to i64) /u 8)) U: [0,4294967289) S: [0,4294967289) ; CHECK-NEXT: %index = phi i64 [ 0, %check ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,8}<%loop> U: [0,25) S: [0,25) Exits: (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,8}<%loop> U: [0,25) S: [0,25) Exits: (-8 + (8 * ((zext i32 %n to i64) /u 8))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw nsw i64 %index, 8 -; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,33) S: [8,33) Exits: (8 + (8 * ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {8,+,8}<%loop> U: [8,33) S: [8,33) Exits: (8 * ((zext i32 %n to i64) /u 8)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_and_base_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * ((zext i32 %n to i64) /u 8))) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -412,9 +412,9 @@ define void @guard_pessimizes_analysis_step2(i1 %c, i32 %N) { ; CHECK-NEXT: %init = phi i64 [ 2, %entry ], [ 4, %bb1 ] ; CHECK-NEXT: --> %init U: [2,5) S: [2,5) ; CHECK-NEXT: %iv = phi i64 [ %iv.next, %loop ], [ %init, %loop.ph ] -; CHECK-NEXT: --> {%init,+,2}<%loop> U: [2,17) S: [2,17) Exits: ((2 * ((14 + (-1 * %init)) /u 2)) + %init) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%init,+,2}<%loop> U: [2,17) S: [2,17) Exits: 14 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = add i64 %iv, 2 -; CHECK-NEXT: --> {(2 + %init),+,2}<%loop> U: [4,19) S: [4,19) Exits: (2 + (2 * ((14 + (-1 * %init)) /u 2)) + %init) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(2 + %init),+,2}<%loop> U: [4,19) S: [4,19) Exits: 16 LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @guard_pessimizes_analysis_step2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((14 + (-1 * %init)) /u 2) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 6 @@ -456,11 +456,11 @@ define i32 @rewrite_sext_slt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_slt_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -500,11 +500,11 @@ define i32 @rewrite_zext_ult_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_ult_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -544,11 +544,11 @@ define i32 @rewrite_zext_ule_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_ule_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((4 umax (zext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -588,11 +588,11 @@ define i32 @rewrite_zext_sle_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_sle_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((zext i32 (4 smax %N) to i64) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -632,11 +632,11 @@ define i32 @rewrite_zext_uge_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * ((16 umin (zext i32 %N to i64)) /u 4)) U: [0,17) S: [0,17) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * ((16 umin (zext i32 %N to i64)) /u 4)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_uge_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -676,11 +676,11 @@ define i32 @rewrite_sext_sge_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_sge_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -720,11 +720,11 @@ define i32 @rewrite_zext_ugt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * ((16 umin (zext i32 %N to i64)) /u 4)) U: [0,17) S: [0,17) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * ((16 umin (zext i32 %N to i64)) /u 4)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_zext_ugt_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * ((16 umin (zext i32 %N to i64)) /u 4))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -764,11 +764,11 @@ define i32 @rewrite_sext_sgt_narrow_check(i32 %N, ptr %arr) { ; CHECK-NEXT: %n.vec = and i64 %ext, 28 ; CHECK-NEXT: --> (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) U: [0,29) S: [0,29) ; CHECK-NEXT: %index = phi i64 [ 0, %loop.ph ], [ %index.next, %loop ] -; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {0,+,4}<%loop> U: [0,13) S: [0,13) Exits: (-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep = getelementptr inbounds i32, ptr %arr, i64 %index ; CHECK-NEXT: --> {%arr,+,16}<%loop> U: full-set S: full-set Exits: ((16 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4)) + %arr) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %index.next = add nuw i64 %index, 4 -; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 + (4 * ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {4,+,4}<%loop> U: [4,17) S: [4,17) Exits: (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_sext_sgt_narrow_check ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (4 * (zext i3 (trunc i64 ((16 smin (sext i32 %N to i64)) /u 4) to i3) to i64))) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 3 @@ -808,9 +808,9 @@ define void @rewrite_add_rec() { ; CHECK-NEXT: %n.vec = and i64 %sub, -2 ; CHECK-NEXT: --> (2 * ({9,+,-1}<%outer.header> /u 2)) U: [0,9) S: [0,9) Exits: 0 LoopDispositions: { %outer.header: Computable, %inner: Invariant } ; CHECK-NEXT: %inner.iv = phi i64 [ 0, %inner.ph ], [ %inner.iv.next, %inner ] -; CHECK-NEXT: --> {0,+,2}<%inner> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: (2 * ((-2 + (2 * ({9,+,-1}<%outer.header> /u 2))) /u 2)) LoopDispositions: { %inner: Computable, %outer.header: Variant } +; CHECK-NEXT: --> {0,+,2}<%inner> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: (-2 + (2 * ({9,+,-1}<%outer.header> /u 2))) LoopDispositions: { %inner: Computable, %outer.header: Variant } ; CHECK-NEXT: %inner.iv.next = add i64 %inner.iv, 2 -; CHECK-NEXT: --> {2,+,2}<%inner> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: (2 + (2 * ((-2 + (2 * ({9,+,-1}<%outer.header> /u 2))) /u 2))) LoopDispositions: { %inner: Computable, %outer.header: Variant } +; CHECK-NEXT: --> {2,+,2}<%inner> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: (2 * ({9,+,-1}<%outer.header> /u 2)) LoopDispositions: { %inner: Computable, %outer.header: Variant } ; CHECK-NEXT: %iv.next = add i64 %iv, 1 ; CHECK-NEXT: --> {1,+,1}<%outer.header> U: [1,11) S: [1,11) Exits: 10 LoopDispositions: { %outer.header: Computable, %inner: Invariant } ; CHECK-NEXT: Determining loop execution counts for: @rewrite_add_rec diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll index 7cdf3a2d5fd58..4024c986dd11d 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll @@ -1390,9 +1390,9 @@ define void @ptr_induction_eq_2(ptr %a, i64 %n) { ; CHECK-NEXT: %b = getelementptr inbounds ptr, ptr %a, i64 %n ; CHECK-NEXT: --> ((8 * %n) + %a) U: full-set S: full-set ; CHECK-NEXT: %ptr.iv = phi ptr [ %ptr.iv.next, %loop ], [ %a, %entry ] -; CHECK-NEXT: --> {%a,+,8}<%loop> U: full-set S: full-set Exits: ((8 * ((-8 + (8 * %n)) /u 8)) + %a) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%a,+,8}<%loop> U: full-set S: full-set Exits: (-8 + (8 * %n) + %a) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %ptr.iv.next = getelementptr inbounds i8, ptr %ptr.iv, i64 8 -; CHECK-NEXT: --> {(8 + %a),+,8}<%loop> U: full-set S: full-set Exits: (8 + (8 * ((-8 + (8 * %n)) /u 8)) + %a) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(8 + %a),+,8}<%loop> U: full-set S: full-set Exits: ((8 * %n) + %a) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @ptr_induction_eq_2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-8 + (8 * %n)) /u 8) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693951 diff --git a/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll b/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll index c79befac2fb1d..1c108bd7318e9 100644 --- a/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll +++ b/llvm/test/Analysis/ScalarEvolution/pr58402-large-number-of-zext-exprs.ll @@ -17,67 +17,67 @@ define i32 @pr58402_large_number_of_zext(ptr %dst) { ; CHECK-NEXT: %add7 = add i32 %i, 4 ; CHECK-NEXT: --> (4 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [4,5) S: [4,5) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i1 = and i32 %add7, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2)) U: [4,5) S: [4,5) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (4 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [4,5) S: [4,5) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.1 = add i32 %i1, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) U: [8,9) S: [8,9) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (8 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [8,9) S: [8,9) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i2 = and i32 %add7.1, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2)) U: [8,9) S: [8,9) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (8 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [8,9) S: [8,9) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.2 = add i32 %i2, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) U: [12,13) S: [12,13) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (12 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [12,13) S: [12,13) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i3 = and i32 %add7.2, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2)) U: [12,13) S: [12,13) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (12 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [12,13) S: [12,13) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.3 = add i32 %i3, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) U: [16,17) S: [16,17) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (16 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [16,17) S: [16,17) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i4 = and i32 %add7.3, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [16,17) S: [16,17) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (16 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [16,17) S: [16,17) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.4 = add i32 %i4, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [20,21) S: [20,21) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (20 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [20,21) S: [20,21) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i5 = and i32 %add7.4, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [20,21) S: [20,21) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (20 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [20,21) S: [20,21) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.5 = add i32 %i5, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [24,25) S: [24,25) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (24 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [24,25) S: [24,25) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i6 = and i32 %add7.5, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [24,25) S: [24,25) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (24 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [24,25) S: [24,25) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.6 = add i32 %i6, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [28,29) S: [28,29) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (28 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [28,29) S: [28,29) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i7 = and i32 %add7.6, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [28,29) S: [28,29) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (28 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [28,29) S: [28,29) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.7 = add i32 %i7, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [32,33) S: [32,33) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (32 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [32,33) S: [32,33) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i8 = and i32 %add7.7, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [32,33) S: [32,33) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (32 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [32,33) S: [32,33) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.8 = add i32 %i8, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [36,37) S: [36,37) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (36 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [36,37) S: [36,37) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i9 = and i32 %add7.8, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [36,37) S: [36,37) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (36 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [36,37) S: [36,37) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.9 = add i32 %i9, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [40,41) S: [40,41) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (40 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [40,41) S: [40,41) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i10 = and i32 %add7.9, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [40,41) S: [40,41) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (40 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [40,41) S: [40,41) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.10 = add i32 %i10, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [44,45) S: [44,45) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (44 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [44,45) S: [44,45) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i11 = and i32 %add7.10, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [44,45) S: [44,45) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (44 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [44,45) S: [44,45) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.11 = add i32 %i11, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [48,49) S: [48,49) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (48 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [48,49) S: [48,49) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i12 = and i32 %add7.11, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [48,49) S: [48,49) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (48 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [48,49) S: [48,49) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.12 = add i32 %i12, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [52,53) S: [52,53) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (52 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [52,53) S: [52,53) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i13 = and i32 %add7.12, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [52,53) S: [52,53) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (52 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [52,53) S: [52,53) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.13 = add i32 %i13, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [56,57) S: [56,57) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (56 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [56,57) S: [56,57) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i14 = and i32 %add7.13, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [56,57) S: [56,57) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (56 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [56,57) S: [56,57) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.14 = add i32 %i14, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [60,61) S: [60,61) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (60 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [60,61) S: [60,61) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i15 = and i32 %add7.14, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [60,61) S: [60,61) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (60 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [60,61) S: [60,61) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %add7.15 = add i32 %i15, 4 -; CHECK-NEXT: --> (4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) U: [64,65) S: [64,65) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (64 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [64,65) S: [64,65) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: %i16 = and i32 %add7.15, -2 -; CHECK-NEXT: --> (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((4 + (2 * ((zext i1 %cmp to i32) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2))) /u 2)) U: [64,65) S: [64,65) Exits: <> LoopDispositions: { %header: Variant } +; CHECK-NEXT: --> (64 + (2 * ((zext i1 %cmp to i32) /u 2))) U: [64,65) S: [64,65) Exits: <> LoopDispositions: { %header: Variant } ; CHECK-NEXT: Determining loop execution counts for: @pr58402_large_number_of_zext ; CHECK-NEXT: Loop %header: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %header: Unpredictable constant max backedge-taken count. diff --git a/llvm/test/CodeGen/PowerPC/common-chain.ll b/llvm/test/CodeGen/PowerPC/common-chain.ll index b71a360d1be12..8283e7bac3457 100644 --- a/llvm/test/CodeGen/PowerPC/common-chain.ll +++ b/llvm/test/CodeGen/PowerPC/common-chain.ll @@ -721,6 +721,13 @@ define signext i32 @spill_reduce_succ(ptr %input1, ptr %input2, ptr %output, i64 ; CHECK-LABEL: spill_reduce_succ: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: cmpdi r6, 0 +; CHECK-NEXT: ble cr0, .LBB7_9 +; CHECK-NEXT: # %bb.1: # %for.body.preheader +; CHECK-NEXT: sldi r6, r6, 2 +; CHECK-NEXT: li r11, 1 +; CHECK-NEXT: std r26, -48(r1) # 8-byte Folded Spill +; CHECK-NEXT: mr r26, r10 +; CHECK-NEXT: cmpdi r6, 1 ; CHECK-NEXT: std r14, -144(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r15, -136(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r16, -128(r1) # 8-byte Folded Spill @@ -733,231 +740,232 @@ define signext i32 @spill_reduce_succ(ptr %input1, ptr %input2, ptr %output, i64 ; CHECK-NEXT: std r23, -72(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r24, -64(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r25, -56(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r26, -48(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r27, -40(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r28, -32(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r29, -24(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r30, -16(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r31, -8(r1) # 8-byte Folded Spill ; CHECK-NEXT: std r2, -152(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r9, -184(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r8, -176(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r7, -168(r1) # 8-byte Folded Spill -; CHECK-NEXT: std r3, -160(r1) # 8-byte Folded Spill -; CHECK-NEXT: ble cr0, .LBB7_7 -; CHECK-NEXT: # %bb.1: # %for.body.preheader -; CHECK-NEXT: sldi r6, r6, 2 -; CHECK-NEXT: li r7, 1 -; CHECK-NEXT: mr r30, r10 -; CHECK-NEXT: cmpdi r6, 1 -; CHECK-NEXT: iselgt r7, r6, r7 -; CHECK-NEXT: addi r8, r7, -1 -; CHECK-NEXT: clrldi r6, r7, 63 -; CHECK-NEXT: cmpldi r8, 3 -; CHECK-NEXT: blt cr0, .LBB7_4 +; CHECK-NEXT: iselgt r11, r6, r11 +; CHECK-NEXT: addi r12, r11, -1 +; CHECK-NEXT: cmpldi r12, 3 +; CHECK-NEXT: clrldi r6, r11, 63 +; CHECK-NEXT: blt cr0, .LBB7_5 ; CHECK-NEXT: # %bb.2: # %for.body.preheader.new -; CHECK-NEXT: ld r14, -168(r1) # 8-byte Folded Reload -; CHECK-NEXT: mulli r24, r30, 24 -; CHECK-NEXT: ld r16, -184(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r15, -176(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r3, -160(r1) # 8-byte Folded Reload -; CHECK-NEXT: rldicl r0, r7, 62, 2 -; CHECK-NEXT: sldi r11, r30, 5 -; CHECK-NEXT: sldi r19, r30, 4 -; CHECK-NEXT: sldi r7, r14, 3 -; CHECK-NEXT: add r14, r30, r14 -; CHECK-NEXT: sldi r10, r16, 3 -; CHECK-NEXT: sldi r12, r15, 3 -; CHECK-NEXT: add r16, r30, r16 -; CHECK-NEXT: add r15, r30, r15 -; CHECK-NEXT: add r27, r11, r7 -; CHECK-NEXT: add r22, r24, r7 -; CHECK-NEXT: add r17, r19, r7 -; CHECK-NEXT: sldi r2, r14, 3 -; CHECK-NEXT: add r26, r24, r10 -; CHECK-NEXT: add r25, r24, r12 -; CHECK-NEXT: add r21, r19, r10 -; CHECK-NEXT: add r20, r19, r12 -; CHECK-NEXT: add r8, r11, r10 -; CHECK-NEXT: sldi r16, r16, 3 -; CHECK-NEXT: add r29, r5, r27 -; CHECK-NEXT: add r28, r4, r27 -; CHECK-NEXT: add r27, r3, r27 -; CHECK-NEXT: add r24, r5, r22 -; CHECK-NEXT: add r23, r4, r22 -; CHECK-NEXT: add r22, r3, r22 -; CHECK-NEXT: add r19, r5, r17 -; CHECK-NEXT: add r18, r4, r17 -; CHECK-NEXT: add r17, r3, r17 -; CHECK-NEXT: add r14, r5, r2 -; CHECK-NEXT: add r31, r4, r2 -; CHECK-NEXT: add r2, r3, r2 -; CHECK-NEXT: add r9, r5, r8 -; CHECK-NEXT: add r8, r11, r12 +; CHECK-NEXT: rldicl r11, r11, 62, 2 +; CHECK-NEXT: sldi r20, r8, 3 +; CHECK-NEXT: mr r14, r7 +; CHECK-NEXT: sldi r7, r7, 3 +; CHECK-NEXT: sldi r21, r9, 3 +; CHECK-NEXT: std r3, -160(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r9, -208(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r8, -184(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r5, -200(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r4, -168(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r11, -192(r1) # 8-byte Folded Spill +; CHECK-NEXT: sldi r11, r10, 5 +; CHECK-NEXT: add r0, r11, r20 +; CHECK-NEXT: add r12, r11, r21 +; CHECK-NEXT: add r30, r5, r0 +; CHECK-NEXT: add r0, r11, r7 +; CHECK-NEXT: std r21, -216(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r20, -224(r1) # 8-byte Folded Spill +; CHECK-NEXT: add r12, r5, r12 +; CHECK-NEXT: add r29, r5, r0 +; CHECK-NEXT: add r28, r4, r0 +; CHECK-NEXT: add r27, r3, r0 +; CHECK-NEXT: mulli r0, r10, 24 +; CHECK-NEXT: std r14, -176(r1) # 8-byte Folded Spill +; CHECK-NEXT: add r26, r0, r21 +; CHECK-NEXT: add r25, r0, r20 +; CHECK-NEXT: add r0, r0, r7 +; CHECK-NEXT: add r24, r5, r0 +; CHECK-NEXT: add r23, r4, r0 +; CHECK-NEXT: add r22, r3, r0 +; CHECK-NEXT: sldi r0, r10, 4 ; CHECK-NEXT: add r26, r5, r26 ; CHECK-NEXT: add r25, r5, r25 +; CHECK-NEXT: add r21, r0, r21 +; CHECK-NEXT: add r20, r0, r20 +; CHECK-NEXT: add r0, r0, r7 +; CHECK-NEXT: add r19, r5, r0 +; CHECK-NEXT: add r18, r4, r0 +; CHECK-NEXT: add r17, r3, r0 +; CHECK-NEXT: add r0, r10, r9 ; CHECK-NEXT: add r21, r5, r21 ; CHECK-NEXT: add r20, r5, r20 -; CHECK-NEXT: add r16, r5, r16 -; CHECK-NEXT: add r8, r5, r8 -; CHECK-NEXT: rldicl r3, r0, 2, 1 -; CHECK-NEXT: addi r3, r3, -4 -; CHECK-NEXT: sub r0, r12, r7 -; CHECK-NEXT: sub r12, r10, r7 -; CHECK-NEXT: li r7, 0 -; CHECK-NEXT: mr r10, r30 -; CHECK-NEXT: sldi r15, r15, 3 -; CHECK-NEXT: add r15, r5, r15 -; CHECK-NEXT: rldicl r3, r3, 62, 2 -; CHECK-NEXT: addi r3, r3, 1 -; CHECK-NEXT: mtctr r3 +; CHECK-NEXT: sldi r0, r0, 3 +; CHECK-NEXT: add r16, r5, r0 +; CHECK-NEXT: add r0, r10, r8 +; CHECK-NEXT: sldi r0, r0, 3 +; CHECK-NEXT: add r15, r5, r0 +; CHECK-NEXT: add r0, r10, r14 +; CHECK-NEXT: sldi r0, r0, 3 +; CHECK-NEXT: add r2, r3, r0 +; CHECK-NEXT: ld r3, -224(r1) # 8-byte Folded Reload +; CHECK-NEXT: add r14, r5, r0 +; CHECK-NEXT: add r31, r4, r0 +; CHECK-NEXT: sub r0, r3, r7 +; CHECK-NEXT: ld r3, -192(r1) # 8-byte Folded Reload +; CHECK-NEXT: rldicl r9, r3, 2, 1 +; CHECK-NEXT: ld r3, -216(r1) # 8-byte Folded Reload +; CHECK-NEXT: addi r8, r9, -4 +; CHECK-NEXT: rldicl r8, r8, 62, 2 +; CHECK-NEXT: sub r7, r3, r7 +; CHECK-NEXT: ori r3, r9, 1 +; CHECK-NEXT: addi r8, r8, 1 +; CHECK-NEXT: mulld r3, r10, r3 +; CHECK-NEXT: mtctr r8 +; CHECK-NEXT: li r8, 0 +; CHECK-NEXT: std r10, -192(r1) # 8-byte Folded Spill +; CHECK-NEXT: std r3, -216(r1) # 8-byte Folded Spill ; CHECK-NEXT: .p2align 4 ; CHECK-NEXT: .LBB7_3: # %for.body ; CHECK-NEXT: # ; CHECK-NEXT: lfd f0, 0(r2) ; CHECK-NEXT: lfd f1, 0(r31) -; CHECK-NEXT: add r3, r10, r30 -; CHECK-NEXT: add r3, r3, r30 ; CHECK-NEXT: xsmuldp f0, f0, f1 ; CHECK-NEXT: lfd f1, 0(r14) -; CHECK-NEXT: add r3, r3, r30 -; CHECK-NEXT: add r10, r3, r30 ; CHECK-NEXT: xsadddp f0, f1, f0 ; CHECK-NEXT: stfd f0, 0(r14) ; CHECK-NEXT: add r14, r14, r11 ; CHECK-NEXT: lfdx f0, r2, r0 ; CHECK-NEXT: lfdx f1, r31, r0 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r15, r7 +; CHECK-NEXT: lfdx f1, r15, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r15, r7 -; CHECK-NEXT: lfdx f0, r2, r12 -; CHECK-NEXT: lfdx f1, r31, r12 +; CHECK-NEXT: stfdx f0, r15, r8 +; CHECK-NEXT: lfdx f0, r2, r7 +; CHECK-NEXT: lfdx f1, r31, r7 ; CHECK-NEXT: add r2, r2, r11 ; CHECK-NEXT: add r31, r31, r11 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r16, r7 +; CHECK-NEXT: lfdx f1, r16, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r16, r7 +; CHECK-NEXT: stfdx f0, r16, r8 ; CHECK-NEXT: lfd f0, 0(r17) ; CHECK-NEXT: lfd f1, 0(r18) ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r19, r7 +; CHECK-NEXT: lfdx f1, r19, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r19, r7 +; CHECK-NEXT: stfdx f0, r19, r8 ; CHECK-NEXT: lfdx f0, r17, r0 ; CHECK-NEXT: lfdx f1, r18, r0 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r20, r7 +; CHECK-NEXT: lfdx f1, r20, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r20, r7 -; CHECK-NEXT: lfdx f0, r17, r12 -; CHECK-NEXT: lfdx f1, r18, r12 +; CHECK-NEXT: stfdx f0, r20, r8 +; CHECK-NEXT: lfdx f0, r17, r7 +; CHECK-NEXT: lfdx f1, r18, r7 ; CHECK-NEXT: add r17, r17, r11 ; CHECK-NEXT: add r18, r18, r11 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r21, r7 +; CHECK-NEXT: lfdx f1, r21, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r21, r7 +; CHECK-NEXT: stfdx f0, r21, r8 ; CHECK-NEXT: lfd f0, 0(r22) ; CHECK-NEXT: lfd f1, 0(r23) ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r24, r7 +; CHECK-NEXT: lfdx f1, r24, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r24, r7 +; CHECK-NEXT: stfdx f0, r24, r8 ; CHECK-NEXT: lfdx f0, r22, r0 ; CHECK-NEXT: lfdx f1, r23, r0 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r25, r7 +; CHECK-NEXT: lfdx f1, r25, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r25, r7 -; CHECK-NEXT: lfdx f0, r22, r12 -; CHECK-NEXT: lfdx f1, r23, r12 +; CHECK-NEXT: stfdx f0, r25, r8 +; CHECK-NEXT: lfdx f0, r22, r7 +; CHECK-NEXT: lfdx f1, r23, r7 ; CHECK-NEXT: add r22, r22, r11 ; CHECK-NEXT: add r23, r23, r11 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r26, r7 +; CHECK-NEXT: lfdx f1, r26, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r26, r7 +; CHECK-NEXT: stfdx f0, r26, r8 ; CHECK-NEXT: lfd f0, 0(r27) ; CHECK-NEXT: lfd f1, 0(r28) ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r29, r7 +; CHECK-NEXT: lfdx f1, r29, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r29, r7 +; CHECK-NEXT: stfdx f0, r29, r8 ; CHECK-NEXT: lfdx f0, r27, r0 ; CHECK-NEXT: lfdx f1, r28, r0 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r8, r7 +; CHECK-NEXT: lfdx f1, r30, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r8, r7 -; CHECK-NEXT: lfdx f0, r27, r12 -; CHECK-NEXT: lfdx f1, r28, r12 +; CHECK-NEXT: stfdx f0, r30, r8 +; CHECK-NEXT: lfdx f0, r27, r7 +; CHECK-NEXT: lfdx f1, r28, r7 ; CHECK-NEXT: add r27, r27, r11 ; CHECK-NEXT: add r28, r28, r11 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r9, r7 +; CHECK-NEXT: lfdx f1, r12, r8 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r9, r7 -; CHECK-NEXT: add r7, r7, r11 +; CHECK-NEXT: stfdx f0, r12, r8 +; CHECK-NEXT: add r8, r8, r11 ; CHECK-NEXT: bdnz .LBB7_3 -; CHECK-NEXT: .LBB7_4: # %for.cond.cleanup.loopexit.unr-lcssa +; CHECK-NEXT: # %bb.4: +; CHECK-NEXT: ld r3, -160(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r4, -168(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r7, -176(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r8, -184(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r10, -192(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r5, -200(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r9, -208(r1) # 8-byte Folded Reload +; CHECK-NEXT: ld r26, -216(r1) # 8-byte Folded Reload +; CHECK-NEXT: .LBB7_5: # %for.cond.cleanup.loopexit.unr-lcssa ; CHECK-NEXT: cmpldi r6, 0 -; CHECK-NEXT: beq cr0, .LBB7_7 -; CHECK-NEXT: # %bb.5: # %for.body.epil.preheader -; CHECK-NEXT: ld r3, -184(r1) # 8-byte Folded Reload -; CHECK-NEXT: ld r0, -160(r1) # 8-byte Folded Reload -; CHECK-NEXT: sldi r8, r30, 3 -; CHECK-NEXT: add r3, r10, r3 -; CHECK-NEXT: sldi r3, r3, 3 -; CHECK-NEXT: add r7, r5, r3 -; CHECK-NEXT: add r9, r4, r3 -; CHECK-NEXT: add r11, r0, r3 -; CHECK-NEXT: ld r3, -176(r1) # 8-byte Folded Reload -; CHECK-NEXT: add r3, r10, r3 -; CHECK-NEXT: sldi r3, r3, 3 -; CHECK-NEXT: add r12, r5, r3 -; CHECK-NEXT: add r30, r4, r3 -; CHECK-NEXT: add r29, r0, r3 -; CHECK-NEXT: ld r3, -168(r1) # 8-byte Folded Reload -; CHECK-NEXT: add r3, r10, r3 -; CHECK-NEXT: li r10, 0 -; CHECK-NEXT: sldi r3, r3, 3 -; CHECK-NEXT: add r5, r5, r3 -; CHECK-NEXT: add r4, r4, r3 -; CHECK-NEXT: add r3, r0, r3 +; CHECK-NEXT: beq cr0, .LBB7_8 +; CHECK-NEXT: # %bb.6: # %for.body.epil.preheader +; CHECK-NEXT: add r11, r26, r9 +; CHECK-NEXT: add r12, r26, r8 +; CHECK-NEXT: add r9, r26, r7 +; CHECK-NEXT: sldi r27, r10, 3 +; CHECK-NEXT: sldi r11, r11, 3 +; CHECK-NEXT: sldi r0, r12, 3 +; CHECK-NEXT: sldi r9, r9, 3 +; CHECK-NEXT: add r28, r5, r11 +; CHECK-NEXT: add r10, r4, r11 +; CHECK-NEXT: add r11, r3, r11 +; CHECK-NEXT: add r12, r5, r0 +; CHECK-NEXT: add r30, r4, r0 +; CHECK-NEXT: add r29, r3, r0 +; CHECK-NEXT: add r5, r5, r9 +; CHECK-NEXT: add r4, r4, r9 +; CHECK-NEXT: add r3, r3, r9 +; CHECK-NEXT: li r9, 0 ; CHECK-NEXT: .p2align 4 -; CHECK-NEXT: .LBB7_6: # %for.body.epil +; CHECK-NEXT: .LBB7_7: # %for.body.epil ; CHECK-NEXT: # -; CHECK-NEXT: lfdx f0, r3, r10 -; CHECK-NEXT: lfdx f1, r4, r10 +; CHECK-NEXT: lfdx f0, r3, r9 +; CHECK-NEXT: lfdx f1, r4, r9 ; CHECK-NEXT: addi r6, r6, -1 ; CHECK-NEXT: cmpldi r6, 0 ; CHECK-NEXT: xsmuldp f0, f0, f1 ; CHECK-NEXT: lfd f1, 0(r5) ; CHECK-NEXT: xsadddp f0, f1, f0 ; CHECK-NEXT: stfd f0, 0(r5) -; CHECK-NEXT: add r5, r5, r8 -; CHECK-NEXT: lfdx f0, r29, r10 -; CHECK-NEXT: lfdx f1, r30, r10 +; CHECK-NEXT: add r5, r5, r27 +; CHECK-NEXT: lfdx f0, r29, r9 +; CHECK-NEXT: lfdx f1, r30, r9 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r12, r10 +; CHECK-NEXT: lfdx f1, r12, r9 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r12, r10 -; CHECK-NEXT: lfdx f0, r11, r10 -; CHECK-NEXT: lfdx f1, r9, r10 +; CHECK-NEXT: stfdx f0, r12, r9 +; CHECK-NEXT: lfdx f0, r11, r9 +; CHECK-NEXT: lfdx f1, r10, r9 ; CHECK-NEXT: xsmuldp f0, f0, f1 -; CHECK-NEXT: lfdx f1, r7, r10 +; CHECK-NEXT: lfdx f1, r28, r9 ; CHECK-NEXT: xsadddp f0, f1, f0 -; CHECK-NEXT: stfdx f0, r7, r10 -; CHECK-NEXT: add r10, r10, r8 -; CHECK-NEXT: bne cr0, .LBB7_6 -; CHECK-NEXT: .LBB7_7: # %for.cond.cleanup +; CHECK-NEXT: stfdx f0, r28, r9 +; CHECK-NEXT: add r9, r9, r27 +; CHECK-NEXT: bne cr0, .LBB7_7 +; CHECK-NEXT: .LBB7_8: ; CHECK-NEXT: ld r2, -152(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r31, -8(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r30, -16(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r29, -24(r1) # 8-byte Folded Reload -; CHECK-NEXT: li r3, 0 ; CHECK-NEXT: ld r28, -32(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r27, -40(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r26, -48(r1) # 8-byte Folded Reload @@ -973,6 +981,8 @@ define signext i32 @spill_reduce_succ(ptr %input1, ptr %input2, ptr %output, i64 ; CHECK-NEXT: ld r16, -128(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r15, -136(r1) # 8-byte Folded Reload ; CHECK-NEXT: ld r14, -144(r1) # 8-byte Folded Reload +; CHECK-NEXT: .LBB7_9: # %for.cond.cleanup +; CHECK-NEXT: li r3, 0 ; CHECK-NEXT: blr entry: %cmp49 = icmp sgt i64 %m, 0 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll index 07aa05f609c40..6f4f03caf8f37 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-asm.ll @@ -912,7 +912,7 @@ define void @strided_load_startval_add_with_splat(ptr noalias nocapture %arg, pt ; CHECK-NEXT: addi a6, a4, 1 ; CHECK-NEXT: andi a7, a6, -32 ; CHECK-NEXT: add a4, a7, a2 -; CHECK-NEXT: add a2, a4, a0 +; CHECK-NEXT: add a2, a0, a4 ; CHECK-NEXT: li t1, 5 ; CHECK-NEXT: vsetvli zero, t2, e8, m1, ta, ma ; CHECK-NEXT: .LBB14_3: # %bb15 @@ -1019,10 +1019,7 @@ define void @gather_no_scalar_remainder(ptr noalias nocapture noundef %arg, ptr ; CHECK-NEXT: slli a2, a2, 4 ; CHECK-NEXT: beqz a2, .LBB15_3 ; CHECK-NEXT: # %bb.1: # %bb2 -; CHECK-NEXT: addi a2, a2, -16 -; CHECK-NEXT: andi a2, a2, -16 -; CHECK-NEXT: add a2, a2, a0 -; CHECK-NEXT: addi a2, a2, 16 +; CHECK-NEXT: add a2, a0, a2 ; CHECK-NEXT: li a3, 5 ; CHECK-NEXT: vsetivli zero, 16, e8, mf2, ta, ma ; CHECK-NEXT: .LBB15_2: # %bb4 diff --git a/llvm/test/Transforms/LoopIdiom/basic.ll b/llvm/test/Transforms/LoopIdiom/basic.ll index 8fdaac3fdffe3..e8ea912246728 100644 --- a/llvm/test/Transforms/LoopIdiom/basic.ll +++ b/llvm/test/Transforms/LoopIdiom/basic.ll @@ -682,10 +682,7 @@ define void @PR14241(ptr %s, i64 %size) { ; CHECK-NEXT: [[END_PTR:%.*]] = getelementptr inbounds i32, ptr [[S:%.*]], i64 [[END_IDX]] ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr nuw i8, ptr [[S]], i64 4 ; CHECK-NEXT: [[TMP0:%.*]] = shl i64 [[SIZE]], 2 -; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], -8 -; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 2 -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP2]], 2 -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], 4 +; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP0]], -4 ; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 4 [[S]], ptr align 4 [[SCEVGEP]], i64 [[TMP4]], i1 false) ; CHECK-NEXT: br label [[WHILE_BODY:%.*]] ; CHECK: while.body: diff --git a/llvm/test/Transforms/LoopStrengthReduce/X86/zext-signed-addrec.ll b/llvm/test/Transforms/LoopStrengthReduce/X86/zext-signed-addrec.ll new file mode 100644 index 0000000000000..f2fa771ac6f29 --- /dev/null +++ b/llvm/test/Transforms/LoopStrengthReduce/X86/zext-signed-addrec.ll @@ -0,0 +1,121 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -loop-reduce -S %s | FileCheck %s +; PR18000 + +target datalayout = "e-i64:64-f80:128-s:64-n8:16:32:64-S128" +target triple = "x86_64-unknown-linux-gnu" + +@a = global i32 0, align 4 +@b = common global i32 0, align 4 +@e = common global i8 0, align 1 +@d = common global i32 0, align 4 +@c = common global i32 0, align 4 +@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1 + +define i32 @foo() { +; CHECK-LABEL: define i32 @foo() { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[DOTPR:%.*]] = load i32, ptr @b, align 4 +; CHECK-NEXT: [[CMP10:%.*]] = icmp slt i32 [[DOTPR]], 1 +; CHECK-NEXT: br i1 [[CMP10]], label %[[OUTER_PH:.*]], label %[[ENTRY_ELSE:.*]] +; CHECK: [[ENTRY_ELSE]]: +; CHECK-NEXT: [[DOTPRE:%.*]] = load i32, ptr @c, align 4 +; CHECK-NEXT: br label %[[MERGE:.*]] +; CHECK: [[OUTER_PH]]: +; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr @a, align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP0]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label %[[OUTER_HEADER_PREHEADER:.*]], label %[[P_ELSE:.*]] +; CHECK: [[OUTER_HEADER_PREHEADER]]: +; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] +; CHECK: [[OUTER_HEADER]]: +; CHECK-NEXT: [[TMP1:%.*]] = phi i32 [ [[INC:%.*]], %[[OUTER_LATCH:.*]] ], [ [[DOTPR]], %[[OUTER_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label %[[INNER_LOOP:.*]] +; CHECK: [[INNER_LOOP]]: +; CHECK-NEXT: [[LSR_IV:%.*]] = phi i32 [ [[LSR_IV_NEXT:%.*]], %[[INNER_LOOP]] ], [ 516, %[[OUTER_HEADER]] ] +; CHECK-NEXT: [[TMP2:%.*]] = phi i8 [ 1, %[[OUTER_HEADER]] ], [ [[DEC:%.*]], %[[INNER_LOOP]] ] +; CHECK-NEXT: [[SHL:%.*]] = add i32 [[LSR_IV]], -258 +; CHECK-NEXT: store i32 [[SHL]], ptr @c, align 4 +; CHECK-NEXT: [[DEC]] = add i8 [[TMP2]], -1 +; CHECK-NEXT: [[LSR_IV_NEXT]] = add nsw i32 [[LSR_IV]], -258 +; CHECK-NEXT: [[CMP2:%.*]] = icmp sgt i8 [[DEC]], -1 +; CHECK-NEXT: br i1 [[CMP2]], label %[[INNER_LOOP]], label %[[OUTER_LATCH]] +; CHECK: [[OUTER_LATCH]]: +; CHECK-NEXT: [[LSR_IV_NEXT_LCSSA:%.*]] = phi i32 [ [[LSR_IV_NEXT]], %[[INNER_LOOP]] ] +; CHECK-NEXT: store i32 0, ptr @d, align 4 +; CHECK-NEXT: [[INC]] = add nsw i32 [[TMP1]], 1 +; CHECK-NEXT: store i32 [[INC]], ptr @b, align 4 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP1]], 0 +; CHECK-NEXT: br i1 [[CMP]], label %[[OUTER_HEADER]], label %[[OUTER_EXIT:.*]] +; CHECK: [[OUTER_EXIT]]: +; CHECK-NEXT: [[LSR_IV_NEXT_LCSSA_LCSSA:%.*]] = phi i32 [ [[LSR_IV_NEXT_LCSSA]], %[[OUTER_LATCH]] ] +; CHECK-NEXT: store i8 [[DEC]], ptr @e, align 1 +; CHECK-NEXT: br label %[[MERGE]] +; CHECK: [[MERGE]]: +; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ [[DOTPRE]], %[[ENTRY_ELSE]] ], [ [[LSR_IV_NEXT_LCSSA_LCSSA]], %[[OUTER_EXIT]] ] +; CHECK-NEXT: [[CALL:%.*]] = tail call i32 @bar(i32 [[TMP3]]) +; CHECK-NEXT: br label %[[RETURN:.*]] +; CHECK: [[P_ELSE]]: +; CHECK-NEXT: store i8 1, ptr @e, align 1 +; CHECK-NEXT: store i32 0, ptr @d, align 4 +; CHECK-NEXT: br label %[[RETURN]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[RETVAL_0:%.*]] = phi i32 [ 0, %[[MERGE]] ], [ 1, %[[P_ELSE]] ] +; CHECK-NEXT: ret i32 [[RETVAL_0]] +; +entry: + %.pr = load i32, ptr @b, align 4 + %cmp10 = icmp slt i32 %.pr, 1 + br i1 %cmp10, label %outer.ph, label %entry.else + +entry.else: + %.pre = load i32, ptr @c, align 4 + br label %merge + +outer.ph: + %0 = load i32, ptr @a, align 4 + %tobool = icmp eq i32 %0, 0 + br i1 %tobool, label %outer.header, label %p.else + +outer.header: + %1 = phi i32 [ %.pr, %outer.ph ], [ %inc, %outer.latch ] + br label %inner.loop + +inner.loop: + %iv = phi i32 [ 1, %outer.header ], [ %iv.next, %inner.loop ] + %2 = phi i8 [ 1, %outer.header ], [ %dec, %inner.loop ] + %conv7 = mul i32 %iv, 258 + %shl = and i32 %conv7, 510 + store i32 %shl, ptr @c, align 4 + %dec = add i8 %2, -1 + %cmp2 = icmp sgt i8 %dec, -1 + %iv.next = add i32 %iv, -1 + br i1 %cmp2, label %inner.loop, label %outer.latch + +outer.latch: + store i32 0, ptr @d, align 4 + %inc = add nsw i32 %1, 1 + store i32 %inc, ptr @b, align 4 + %cmp = icmp slt i32 %1, 0 + br i1 %cmp, label %outer.header, label %outer.exit + +outer.exit: + store i8 %dec, ptr @e, align 1 + br label %merge + +merge: + %3 = phi i32 [ %.pre, %entry.else ], [ %shl, %outer.exit ] + %call = tail call i32 @bar(i32 %3) + br label %return + +p.else: + store i8 1, ptr @e, align 1 + store i32 0, ptr @d, align 4 + br label %return + +return: + %retval.0 = phi i32 [ 0, %merge ], [ 1, %p.else ] + ret i32 %retval.0 +} + +declare i32 @bar(i32) + diff --git a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll index c4fc60908c7e0..132e77951abd6 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/gather_scatter.ll @@ -597,7 +597,7 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; AVX512-LABEL: @test_gather_not_profitable_pr48429( ; AVX512-NEXT: entry: ; AVX512-NEXT: [[IDX_EXT:%.*]] = sext i32 [[D:%.*]] to i64 -; AVX512-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds float, ptr [[PTR:%.*]], i64 [[IDX_EXT]] +; AVX512-NEXT: [[SCEVGEP1:%.*]] = getelementptr float, ptr [[PTR:%.*]], i64 [[IDX_EXT]] ; AVX512-NEXT: [[CMP_NOT10:%.*]] = icmp eq i32 [[D]], 0 ; AVX512-NEXT: br i1 [[CMP_NOT10]], label [[FOR_END:%.*]], label [[ITER_CHECK:%.*]] ; AVX512: iter.check: @@ -616,17 +616,12 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; AVX512-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 6 ; AVX512-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 8 ; AVX512-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DEST:%.*]], i64 [[TMP8]] -; AVX512-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2 -; AVX512-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 4 -; AVX512-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP10]] ; AVX512-NEXT: [[TMP11:%.*]] = mul nsw i64 [[IDX_EXT]], -4 ; AVX512-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP11]] -; AVX512-NEXT: [[TMP12:%.*]] = sub i64 [[TMP10]], [[TMP4]] -; AVX512-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP12]] ; AVX512-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DEST]], [[SCEVGEP1]] ; AVX512-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR]], [[SCEVGEP]] ; AVX512-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; AVX512-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DEST]], [[SCEVGEP3]] +; AVX512-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DEST]], [[PTR]] ; AVX512-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP]] ; AVX512-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]] ; AVX512-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]] @@ -651,9 +646,9 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x float>, ptr [[TMP18]], align 4, !alias.scope [[META8:![0-9]+]] ; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD]], <16 x ptr> [[TMP14]], i32 4, <16 x i1> splat (i1 true)), !alias.scope [[META11:![0-9]+]], !noalias [[META13:![0-9]+]] ; AVX512-NEXT: [[TMP19:%.*]] = getelementptr float, ptr [[TMP16]], i32 0 -; AVX512-NEXT: [[WIDE_LOAD8:%.*]] = load <16 x float>, ptr [[TMP19]], align 4, !alias.scope [[META15:![0-9]+]] +; AVX512-NEXT: [[WIDE_LOAD6:%.*]] = load <16 x float>, ptr [[TMP19]], align 4, !alias.scope [[META15:![0-9]+]] ; AVX512-NEXT: [[TMP20:%.*]] = getelementptr inbounds float, <16 x ptr> [[TMP14]], i64 1 -; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD8]], <16 x ptr> [[TMP20]], i32 4, <16 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]] +; AVX512-NEXT: call void @llvm.masked.scatter.v16f32.v16p0(<16 x float> [[WIDE_LOAD6]], <16 x ptr> [[TMP20]], i32 4, <16 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]] ; AVX512-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 ; AVX512-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 1024 ; AVX512-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -687,16 +682,16 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; AVX512-NEXT: [[TMP28:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[OFFSET_IDX21]] ; AVX512-NEXT: [[TMP29:%.*]] = getelementptr inbounds float, ptr [[TMP28]], i64 [[IDXPROM]] ; AVX512-NEXT: [[TMP30:%.*]] = getelementptr inbounds float, ptr [[TMP29]], i32 0 -; AVX512-NEXT: [[WIDE_LOAD17:%.*]] = load <8 x float>, ptr [[TMP30]], align 4, !alias.scope [[META17:![0-9]+]] -; AVX512-NEXT: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD17]], <8 x ptr> [[TMP26]], i32 4, <8 x i1> splat (i1 true)), !alias.scope [[META20:![0-9]+]], !noalias [[META22:![0-9]+]] +; AVX512-NEXT: [[WIDE_LOAD14:%.*]] = load <8 x float>, ptr [[TMP30]], align 4, !alias.scope [[META8]] +; AVX512-NEXT: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD14]], <8 x ptr> [[TMP26]], i32 4, <8 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]] ; AVX512-NEXT: [[TMP31:%.*]] = getelementptr float, ptr [[TMP28]], i32 0 -; AVX512-NEXT: [[WIDE_LOAD18:%.*]] = load <8 x float>, ptr [[TMP31]], align 4, !alias.scope [[META24:![0-9]+]] -; AVX512-NEXT: [[TMP32:%.*]] = getelementptr inbounds float, <8 x ptr> [[TMP26]], i64 1 -; AVX512-NEXT: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD18]], <8 x ptr> [[TMP32]], i32 4, <8 x i1> splat (i1 true)), !alias.scope [[META20]], !noalias [[META22]] +; AVX512-NEXT: [[WIDE_LOAD15:%.*]] = load <8 x float>, ptr [[TMP31]], align 4, !alias.scope [[META15]] +; AVX512-NEXT: [[TMP27:%.*]] = getelementptr inbounds float, <8 x ptr> [[TMP26]], i64 1 +; AVX512-NEXT: call void @llvm.masked.scatter.v8f32.v8p0(<8 x float> [[WIDE_LOAD15]], <8 x ptr> [[TMP27]], i32 4, <8 x i1> splat (i1 true)), !alias.scope [[META11]], !noalias [[META13]] ; AVX512-NEXT: [[INDEX_NEXT24]] = add nuw i64 [[INDEX18]], 8 ; AVX512-NEXT: [[PTR_IND20]] = getelementptr i8, ptr [[POINTER_PHI19]], i64 512 -; AVX512-NEXT: [[TMP33:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC10]] -; AVX512-NEXT: br i1 [[TMP33]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP25:![0-9]+]] +; AVX512-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT24]], [[N_VEC10]] +; AVX512-NEXT: br i1 [[TMP32]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] ; AVX512: vec.epilog.middle.block: ; AVX512-NEXT: [[CMP_N17:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC10]] ; AVX512-NEXT: br i1 [[CMP_N17]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]] @@ -715,15 +710,15 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; AVX512-NEXT: store float [[TMP35]], ptr [[ARRAYIDX5]], align 4 ; AVX512-NEXT: [[INCDEC_PTR]] = getelementptr inbounds float, ptr [[PTR_ADDR_012]], i64 1 ; AVX512-NEXT: [[ADD_PTR6]] = getelementptr inbounds float, ptr [[DEST_ADDR_011]], i64 16 -; AVX512-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[INCDEC_PTR]], [[ADD_PTR]] -; AVX512-NEXT: br i1 [[CMP_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] +; AVX512-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[INCDEC_PTR]], [[SCEVGEP1]] +; AVX512-NEXT: br i1 [[CMP_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; AVX512: for.end: ; AVX512-NEXT: ret void ; ; FVW2-LABEL: @test_gather_not_profitable_pr48429( ; FVW2-NEXT: entry: ; FVW2-NEXT: [[IDX_EXT:%.*]] = sext i32 [[D:%.*]] to i64 -; FVW2-NEXT: [[ADD_PTR:%.*]] = getelementptr inbounds float, ptr [[PTR:%.*]], i64 [[IDX_EXT]] +; FVW2-NEXT: [[SCEVGEP1:%.*]] = getelementptr float, ptr [[PTR:%.*]], i64 [[IDX_EXT]] ; FVW2-NEXT: [[CMP_NOT10:%.*]] = icmp eq i32 [[D]], 0 ; FVW2-NEXT: br i1 [[CMP_NOT10]], label [[FOR_END:%.*]], label [[FOR_BODY_LR_PH:%.*]] ; FVW2: for.body.lr.ph: @@ -742,17 +737,12 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; FVW2-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 6 ; FVW2-NEXT: [[TMP8:%.*]] = add nuw nsw i64 [[TMP7]], 8 ; FVW2-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DEST:%.*]], i64 [[TMP8]] -; FVW2-NEXT: [[TMP9:%.*]] = shl nuw i64 [[TMP6]], 2 -; FVW2-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 4 -; FVW2-NEXT: [[SCEVGEP1:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP10]] ; FVW2-NEXT: [[TMP11:%.*]] = mul nsw i64 [[IDX_EXT]], -4 ; FVW2-NEXT: [[SCEVGEP2:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP11]] -; FVW2-NEXT: [[TMP12:%.*]] = sub i64 [[TMP10]], [[TMP4]] -; FVW2-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[PTR]], i64 [[TMP12]] ; FVW2-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DEST]], [[SCEVGEP1]] ; FVW2-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[PTR]], [[SCEVGEP]] ; FVW2-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; FVW2-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DEST]], [[SCEVGEP3]] +; FVW2-NEXT: [[BOUND04:%.*]] = icmp ult ptr [[DEST]], [[PTR]] ; FVW2-NEXT: [[BOUND15:%.*]] = icmp ult ptr [[SCEVGEP2]], [[SCEVGEP]] ; FVW2-NEXT: [[FOUND_CONFLICT6:%.*]] = and i1 [[BOUND04]], [[BOUND15]] ; FVW2-NEXT: [[CONFLICT_RDX:%.*]] = or i1 [[FOUND_CONFLICT]], [[FOUND_CONFLICT6]] @@ -810,7 +800,7 @@ define void @test_gather_not_profitable_pr48429(i32 %d, ptr readonly noalias %pt ; FVW2-NEXT: store float [[TMP32]], ptr [[ARRAYIDX5]], align 4 ; FVW2-NEXT: [[INCDEC_PTR]] = getelementptr inbounds float, ptr [[PTR_ADDR_012]], i64 1 ; FVW2-NEXT: [[ADD_PTR6]] = getelementptr inbounds float, ptr [[DEST_ADDR_011]], i64 16 -; FVW2-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[INCDEC_PTR]], [[ADD_PTR]] +; FVW2-NEXT: [[CMP_NOT:%.*]] = icmp eq ptr [[INCDEC_PTR]], [[SCEVGEP1]] ; FVW2-NEXT: br i1 [[CMP_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] ; FVW2: for.end: ; FVW2-NEXT: ret void diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index a591423e6fc91..74b47a1a263cd 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -318,24 +318,60 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero_i16_p ; CHECK-NEXT: entry: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 2) ] ; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 -; CHECK-NEXT: [[MUL:%.*]] = shl nuw nsw i64 [[N_EXT]], 1 +; CHECK-NEXT: [[MUL:%.*]] = shl i64 [[N_EXT]], 1 ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[A]], i64 [[MUL]]) ] ; CHECK-NEXT: [[A_END:%.*]] = getelementptr i8, ptr [[A]], i64 [[MUL]] ; CHECK-NEXT: [[PRE:%.*]] = icmp eq i32 [[N]], 0 ; CHECK-NEXT: br i1 [[PRE]], label [[EXIT:%.*]], label [[LOOP_HEADER_PREHEADER:%.*]] ; CHECK: loop.header.preheader: +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[MUL]], -2 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw i64 [[TMP1]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-NEXT: [[IV_NEXT1:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP3]] ; CHECK-NEXT: br label [[LOOP_HEADER1:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_HEADER1]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP6]], align 2 +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] +; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP_HEADER1]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP7]], label [[VECTOR_EARLY_EXIT:%.*]], label [[LOOP_LATCH1:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true) +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP12]] +; CHECK-NEXT: br label [[EXIT_LOOPEXIT]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[IV1:%.*]] = phi ptr [ [[IV_NEXT1]], [[LOOP_LATCH1]] ], [ [[A]], [[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] ; CHECK: loop.header: -; CHECK-NEXT: [[IV1:%.*]] = phi ptr [ [[IV_NEXT1:%.*]], [[LOOP_LATCH1:%.*]] ], [ [[A]], [[LOOP_HEADER_PREHEADER]] ] -; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[IV1]], align 2 +; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ [[IV1]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[IV]], align 2 ; CHECK-NEXT: [[C_0:%.*]] = icmp eq i16 [[L]], 0 -; CHECK-NEXT: br i1 [[C_0]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_LATCH1]] +; CHECK-NEXT: br i1 [[C_0]], label [[EXIT_LOOPEXIT]], label [[LOOP_LATCH]] ; CHECK: loop.latch: -; CHECK-NEXT: [[IV_NEXT1]] = getelementptr inbounds nuw i8, ptr [[IV1]], i64 2 -; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[IV_NEXT1]], [[A_END]] -; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER1]] +; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[IV]], i64 2 +; CHECK-NEXT: [[EC:%.*]] = icmp eq ptr [[IV_NEXT]], [[A_END]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: exit.loopexit: -; CHECK-NEXT: [[P_PH:%.*]] = phi ptr [ [[A_END]], [[LOOP_LATCH1]] ], [ [[IV1]], [[LOOP_HEADER1]] ] +; CHECK-NEXT: [[P_PH:%.*]] = phi ptr [ [[A_END]], [[LOOP_LATCH]] ], [ [[IV]], [[LOOP_HEADER]] ], [ [[A_END]], [[LOOP_LATCH1]] ], [ [[TMP13]], [[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: br label [[EXIT]] ; CHECK: exit: ; CHECK-NEXT: [[P:%.*]] = phi ptr [ [[A]], [[ENTRY:%.*]] ], [ [[P_PH]], [[EXIT_LOOPEXIT]] ] From 4be0a04c673d5d335d80c36abfdf2110114c54a6 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 25 Aug 2025 18:55:23 +0100 Subject: [PATCH 25/60] [VPlan] Make VPInstruction::AnyOf poison-safe. (#154156) AnyOf reduces multiple input vectors to a single boolean value. When used for early-exit vectorization, we need to consider any lane after the early exit being poison. Any poison lane would result in poison after the AnyOf reduction. To prevent this, freeze all inputs to AnyOf. Fixes https://github.com/llvm/llvm-project/issues/153946. Fixes https://github.com/llvm/llvm-project/issues/155162. https://alive2.llvm.org/ce/z/FD-XxA PR: https://github.com/llvm/llvm-project/pull/154156 (cherry picked from commit f492eb9509dea311d4e5a5ebbebbf58eb2e7e877) --- llvm/lib/Transforms/Vectorize/VPlan.h | 3 +- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 4 +- .../AArch64/fmax-without-fast-math-flags.ll | 4 +- .../AArch64/fmin-without-fast-math-flags.ll | 4 +- .../AArch64/simple_early_exit.ll | 18 +- .../AArch64/single-early-exit-interleave.ll | 23 +- ...fmax-without-fast-math-flags-interleave.ll | 4 +- .../fmax-without-fast-math-flags.ll | 12 +- .../fmin-without-fast-math-flags.ll | 6 +- .../single-early-exit-cond-poison.ll | 124 +++++++++++ .../single-early-exit-deref-assumptions.ll | 9 +- .../single-early-exit-interleave-hint.ll | 3 +- .../single-early-exit-interleave-only.ll | 91 ++++++++ .../single-early-exit-interleave.ll | 27 ++- .../LoopVectorize/single_early_exit.ll | 18 +- .../single_early_exit_live_outs.ll | 78 ++++--- ...or-loop-backedge-elimination-early-exit.ll | 202 +++++++++++++++--- .../PhaseOrdering/AArch64/std-find.ll | 137 ++++++++++++ 18 files changed, 667 insertions(+), 100 deletions(-) create mode 100644 llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll create mode 100644 llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll create mode 100644 llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 703cfe969577d..b9cd3635047ac 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -994,7 +994,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, // Returns a scalar boolean value, which is true if any lane of its // (boolean) vector operands is true. It produces the reduced value across // all unrolled iterations. Unrolling will add all copies of its original - // operand as additional operands. + // operand as additional operands. AnyOf is poison-safe as all operands + // will be frozen. AnyOf, // Calculates the first active lane index of the vector predicate operands. // It produces the lane index across all unrolled iterations. Unrolling will diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 57b713d3dfcb9..8c424e10a74a5 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -858,9 +858,9 @@ Value *VPInstruction::generate(VPTransformState &State) { return Builder.CreatePtrAdd(Ptr, Addend, Name, getGEPNoWrapFlags()); } case VPInstruction::AnyOf: { - Value *Res = State.get(getOperand(0)); + Value *Res = Builder.CreateFreeze(State.get(getOperand(0))); for (VPValue *Op : drop_begin(operands())) - Res = Builder.CreateOr(Res, State.get(Op)); + Res = Builder.CreateOr(Res, Builder.CreateFreeze(State.get(Op))); return State.VF.isScalar() ? Res : Builder.CreateOrReduce(Res); } case VPInstruction::FirstActiveLane: { diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll index 427a05cc1c843..2b60480afa476 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmax-without-fast-math-flags.ll @@ -63,7 +63,9 @@ define float @fmaxnum(ptr %src, i64 %n) { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] ; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]] -; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP18]], [[TMP15]] ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll index 1a8e5940d88e7..3c091ceb17144 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/fmin-without-fast-math-flags.ll @@ -63,7 +63,9 @@ define float @fminnum(ptr %src, i64 %n) { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] ; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]] -; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP15]], [[TMP18]] ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll index 3bb20e2a81d83..bd84494d6c6c5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/simple_early_exit.ll @@ -37,7 +37,8 @@ define i64 @same_exit_block_pre_inc_use1() #1 { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP14]], align 1 ; CHECK-NEXT: [[TMP16:%.*]] = icmp ne [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP5]] -; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1( [[TMP16]]) +; CHECK-NEXT: [[TMP8:%.*]] = freeze [[TMP16]] +; CHECK-NEXT: [[TMP17:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1( [[TMP8]]) ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]] ; CHECK-NEXT: [[TMP19:%.*]] = or i1 [[TMP17]], [[TMP18]] ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -118,7 +119,8 @@ define i64 @same_exit_block_pre_inc_use4() { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i64>, ptr [[TMP2]], align 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp uge <2 x i64> [[VEC_IND]], [[WIDE_LOAD]] ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 2 -; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <2 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v2i1(<2 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64 ; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <2 x i64> [[VEC_IND]], splat (i64 2) @@ -194,7 +196,8 @@ define i64 @loop_contains_safe_call() #1 { ; CHECK-NEXT: [[TMP3:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]]) ; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast oge <4 x float> [[TMP3]], splat (float 3.000000e+00) ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64 ; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -281,7 +284,8 @@ define i64 @loop_contains_safe_div() #1 { ; CHECK-NEXT: [[TMP13:%.*]] = udiv [[WIDE_LOAD]], splat (i32 20000) ; CHECK-NEXT: [[TMP15:%.*]] = icmp ne [[TMP13]], splat (i32 1) ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX2]], [[TMP5]] -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1( [[TMP15]]) +; CHECK-NEXT: [[TMP9:%.*]] = freeze [[TMP15]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.nxv4i1( [[TMP9]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[INDEX1]] ; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -363,7 +367,8 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align( ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0 ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8 ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -464,7 +469,8 @@ define i32 @diff_exit_block_needs_scev_check(i32 %end) { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4 ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]]) +; CHECK-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP14]] +; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP20]]) ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll index 61ef3cef603fa..21cecea735ad5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/single-early-exit-interleave.ll @@ -29,24 +29,25 @@ define i64 @same_exit_block_pre_inc_use1() #0 { ; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i64 3, [[INDEX1]] ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[OFFSET_IDX]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load , ptr [[TMP8]], align 1 -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load , ptr [[TMP10]], align 1 -; CHECK-NEXT: [[TMP32:%.*]] = icmp ne [[WIDE_LOAD4]], [[WIDE_LOAD8]] +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD4:%.*]] = load , ptr [[TMP28]], align 1 +; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[TMP29]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD8:%.*]] = load , ptr [[TMP26]], align 1 +; CHECK-NEXT: [[TMP59:%.*]] = icmp ne [[WIDE_LOAD4]], [[WIDE_LOAD8]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], [[TMP5]] -; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1( [[TMP32]]) -; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]] -; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]] -; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP34:%.*]] = freeze [[TMP59]] +; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.nxv16i1( [[TMP34]]) +; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]] +; CHECK-NEXT: [[TMP36:%.*]] = or i1 [[TMP12]], [[TMP35]] +; CHECK-NEXT: br i1 [[TMP36]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.split: ; CHECK-NEXT: br i1 [[TMP12]], label [[VECTOR_EARLY_EXIT:%.*]], label [[LOOP_INC:%.*]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 510, [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[LOOP_END:%.*]], label [[SCALAR_PH]] ; CHECK: vector.early.exit: -; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1( [[TMP32]], i1 true) +; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.nxv16i1( [[TMP59]], i1 true) ; CHECK-NEXT: [[TMP16:%.*]] = add i64 [[INDEX1]], [[TMP61]] ; CHECK-NEXT: [[TMP17:%.*]] = add i64 3, [[TMP16]] ; CHECK-NEXT: br label [[LOOP_END]] diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll index a2eddad179216..8b95df4b96c48 100644 --- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll +++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags-interleave.ll @@ -63,7 +63,9 @@ define float @fmaxnum(ptr %src, i64 %n) { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP3:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] ; CHECK-NEXT: [[TMP4:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD2]], [[WIDE_LOAD2]] -; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP3]], [[TMP4]] +; CHECK-NEXT: [[TMP15:%.*]] = freeze <4 x i1> [[TMP3]] +; CHECK-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = or <4 x i1> [[TMP15]], [[TMP18]] ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer diff --git a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll index 1ca5586942d7c..74ea22e0f8d6f 100644 --- a/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/fmax-without-fast-math-flags.ll @@ -208,7 +208,8 @@ define float @fmaxnum_1(ptr %src, i64 %n) { ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]] @@ -276,7 +277,8 @@ define float @fmaxnum_2(ptr %src, i64 %n) { ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]] @@ -346,7 +348,8 @@ define float @fmaxnum_induction_starts_at_10(ptr %src, i64 %n) { ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]] @@ -417,7 +420,8 @@ define float @fmaxnum_induction_starts_at_value(ptr %src, i64 %start, i64 %n) { ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP5:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP6]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP6]], [[TMP4]] diff --git a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll index 68bc8d0640a3f..6b1e411c3ca68 100644 --- a/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll +++ b/llvm/test/Transforms/LoopVectorize/fmin-without-fast-math-flags.ll @@ -208,7 +208,8 @@ define float @fminnum_1(ptr %src, i64 %n) { ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]] @@ -276,7 +277,8 @@ define float @fminnum_2(ptr %src, i64 %n) { ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP2:%.*]] = fcmp uno <4 x float> [[WIDE_LOAD]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[TMP3]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP3]], [[TMP5]] diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll new file mode 100644 index 0000000000000..194d16e41f921 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-cond-poison.ll @@ -0,0 +1,124 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5 +; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=2 -S %s | FileCheck --check-prefix=VF4IC2 %s +; RUN: opt -p loop-vectorize -force-vector-width=8 -force-vector-interleave=1 -S %s | FileCheck --check-prefix=VF8IC1 %s + +; Test case from https://github.com/llvm/llvm-project/issues/153946. +; %shr and thus %early.cond will be poison from %iv == 4 onwards. +; Make sure the mask being poison does not propagate across lanes in the +; OR reduction when computing the early exit condition in the vector loop. +define noundef i32 @f(i32 noundef %g) { +; VF4IC2-LABEL: define noundef i32 @f( +; VF4IC2-SAME: i32 noundef [[G:%.*]]) { +; VF4IC2-NEXT: [[ENTRY:.*]]: +; VF4IC2-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; VF4IC2: [[VECTOR_PH]]: +; VF4IC2-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[G]], i64 0 +; VF4IC2-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer +; VF4IC2-NEXT: br label %[[VECTOR_BODY:.*]] +; VF4IC2: [[VECTOR_BODY]]: +; VF4IC2-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF4IC2-NEXT: [[VEC_IND:%.*]] = phi <4 x i32> [ , %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF4IC2-NEXT: [[TMP0:%.*]] = shl nuw nsw <4 x i32> [[VEC_IND]], splat (i32 3) +; VF4IC2-NEXT: [[TMP3:%.*]] = ashr <4 x i32> [[BROADCAST_SPLAT]], [[TMP0]] +; VF4IC2-NEXT: [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP3]], zeroinitializer +; VF4IC2-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 4 +; VF4IC2-NEXT: [[TMP18:%.*]] = freeze <4 x i1> [[TMP5]] +; VF4IC2-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP18]]) +; VF4IC2-NEXT: [[TMP7:%.*]] = icmp eq i32 [[INDEX_NEXT]], 8 +; VF4IC2-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP7]] +; VF4IC2-NEXT: [[VEC_IND_NEXT]] = add <4 x i32> [[VEC_IND]], splat (i32 4) +; VF4IC2-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; VF4IC2: [[MIDDLE_SPLIT]]: +; VF4IC2-NEXT: [[TMP8:%.*]] = extractelement <4 x i32> [[TMP3]], i32 3 +; VF4IC2-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF4IC2: [[MIDDLE_BLOCK]]: +; VF4IC2-NEXT: br i1 true, label %[[RETURN:.*]], label %[[SCALAR_PH]] +; VF4IC2: [[VECTOR_EARLY_EXIT]]: +; VF4IC2-NEXT: [[TMP14:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP5]], i1 true) +; VF4IC2-NEXT: [[TMP15:%.*]] = trunc i64 [[TMP14]] to i32 +; VF4IC2-NEXT: [[TMP10:%.*]] = add i32 [[INDEX]], [[TMP15]] +; VF4IC2-NEXT: br label %[[RETURN]] +; VF4IC2: [[SCALAR_PH]]: +; VF4IC2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; VF4IC2-NEXT: br label %[[LOOP_HEADER:.*]] +; VF4IC2: [[LOOP_HEADER]]: +; VF4IC2-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; VF4IC2-NEXT: [[MUL:%.*]] = shl nuw nsw i32 [[IV]], 3 +; VF4IC2-NEXT: [[SHR:%.*]] = ashr i32 [[G]], [[MUL]] +; VF4IC2-NEXT: [[EARLY_COND:%.*]] = icmp eq i32 [[SHR]], 0 +; VF4IC2-NEXT: br i1 [[EARLY_COND]], label %[[LOOP_LATCH]], label %[[RETURN]] +; VF4IC2: [[LOOP_LATCH]]: +; VF4IC2-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; VF4IC2-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 8 +; VF4IC2-NEXT: br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]] +; VF4IC2: [[RETURN]]: +; VF4IC2-NEXT: [[RES:%.*]] = phi i32 [ [[SHR]], %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP8]], %[[MIDDLE_BLOCK]] ], [ [[TMP10]], %[[VECTOR_EARLY_EXIT]] ] +; VF4IC2-NEXT: ret i32 [[RES]] +; +; VF8IC1-LABEL: define noundef i32 @f( +; VF8IC1-SAME: i32 noundef [[G:%.*]]) { +; VF8IC1-NEXT: [[ENTRY:.*]]: +; VF8IC1-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; VF8IC1: [[VECTOR_PH]]: +; VF8IC1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i32> poison, i32 [[G]], i64 0 +; VF8IC1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i32> [[BROADCAST_SPLATINSERT]], <8 x i32> poison, <8 x i32> zeroinitializer +; VF8IC1-NEXT: br label %[[VECTOR_BODY:.*]] +; VF8IC1: [[VECTOR_BODY]]: +; VF8IC1-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF8IC1-NEXT: [[VEC_IND:%.*]] = phi <8 x i32> [ , %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF8IC1-NEXT: [[TMP0:%.*]] = shl nuw nsw <8 x i32> [[VEC_IND]], splat (i32 3) +; VF8IC1-NEXT: [[TMP1:%.*]] = ashr <8 x i32> [[BROADCAST_SPLAT]], [[TMP0]] +; VF8IC1-NEXT: [[TMP2:%.*]] = icmp ne <8 x i32> [[TMP1]], zeroinitializer +; VF8IC1-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], 8 +; VF8IC1-NEXT: [[TMP8:%.*]] = freeze <8 x i1> [[TMP2]] +; VF8IC1-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP8]]) +; VF8IC1-NEXT: [[VEC_IND_NEXT]] = add <8 x i32> [[VEC_IND]], splat (i32 8) +; VF8IC1-NEXT: br i1 true, label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; VF8IC1: [[MIDDLE_SPLIT]]: +; VF8IC1-NEXT: [[TMP4:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7 +; VF8IC1-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF8IC1: [[MIDDLE_BLOCK]]: +; VF8IC1-NEXT: br i1 true, label %[[RETURN:.*]], label %[[SCALAR_PH]] +; VF8IC1: [[VECTOR_EARLY_EXIT]]: +; VF8IC1-NEXT: [[TMP5:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP2]], i1 true) +; VF8IC1-NEXT: [[TMP6:%.*]] = trunc i64 [[TMP5]] to i32 +; VF8IC1-NEXT: [[TMP9:%.*]] = add i32 [[INDEX]], [[TMP6]] +; VF8IC1-NEXT: br label %[[RETURN]] +; VF8IC1: [[SCALAR_PH]]: +; VF8IC1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ 8, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; VF8IC1-NEXT: br label %[[LOOP_HEADER:.*]] +; VF8IC1: [[LOOP_HEADER]]: +; VF8IC1-NEXT: [[IV:%.*]] = phi i32 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; VF8IC1-NEXT: [[MUL:%.*]] = shl nuw nsw i32 [[IV]], 3 +; VF8IC1-NEXT: [[SHR:%.*]] = ashr i32 [[G]], [[MUL]] +; VF8IC1-NEXT: [[EARLY_COND:%.*]] = icmp eq i32 [[SHR]], 0 +; VF8IC1-NEXT: br i1 [[EARLY_COND]], label %[[LOOP_LATCH]], label %[[RETURN]] +; VF8IC1: [[LOOP_LATCH]]: +; VF8IC1-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; VF8IC1-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 8 +; VF8IC1-NEXT: br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]] +; VF8IC1: [[RETURN]]: +; VF8IC1-NEXT: [[RES:%.*]] = phi i32 [ [[SHR]], %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ], [ [[TMP4]], %[[MIDDLE_BLOCK]] ], [ [[TMP9]], %[[VECTOR_EARLY_EXIT]] ] +; VF8IC1-NEXT: ret i32 [[RES]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %mul = shl nuw nsw i32 %iv, 3 + %shr = ashr i32 %g, %mul + %early.cond = icmp eq i32 %shr, 0 + br i1 %early.cond, label %loop.latch, label %return + +loop.latch: + %iv.next = add nuw nsw i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 8 + br i1 %ec, label %return, label %loop.header + +return: + %res = phi i32 [ %shr, %loop.latch ], [ %iv, %loop.header ] + ret i32 %res +} + + diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index 74b47a1a263cd..a07e0069efbc4 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -20,7 +20,8 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_si ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024 ; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP]], !llvm.loop [[LOOP0:![0-9]+]] @@ -147,7 +148,8 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero(ptr n ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT3]], [[N_VEC]] ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] ; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -343,7 +345,8 @@ define i64 @early_exit_alignment_and_deref_known_via_assumption_n_not_zero_i16_p ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP6]], align 2 ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP14:%.*]] = freeze <4 x i1> [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP_HEADER1]], !llvm.loop [[LOOP6:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll index de8a3c5a8eaf2..9bd26841146fb 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-hint.ll @@ -23,7 +23,8 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() { ; VF4IC4-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4 ; VF4IC4-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], splat (i32 10) ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; VF4IC4-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP2]] +; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) ; VF4IC4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 ; VF4IC4-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; VF4IC4-NEXT: br i1 [[TMP5]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll new file mode 100644 index 0000000000000..aeb2ad5df46fa --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave-only.ll @@ -0,0 +1,91 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 5 +; RUN: opt -p loop-vectorize -force-vector-width=1 -force-vector-interleave=2 -S %s | FileCheck %s + + +; FIXME: currently the live-outs are not handled correctly. +; Test for https://github.com/llvm/llvm-project/issues/154967. +define i8 @iv_used_in_exit_with_math(i8 noundef %g) { +; CHECK-LABEL: define i8 @iv_used_in_exit_with_math( +; CHECK-SAME: i8 noundef [[G:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[S:%.*]] = shl nuw i8 1, [[IV]] +; CHECK-NEXT: [[A:%.*]] = and i8 [[S]], [[G]] +; CHECK-NEXT: [[C:%.*]] = icmp eq i8 [[A]], 0 +; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[RETURN:.*]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i8 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i8 [[IV_NEXT]], 4 +; CHECK-NEXT: br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[RES_IV1:%.*]] = phi i8 [ 32, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ] +; CHECK-NEXT: [[RES_IV2:%.*]] = phi i8 [ 0, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ] +; CHECK-NEXT: [[RES:%.*]] = add i8 [[RES_IV1]], [[RES_IV2]] +; CHECK-NEXT: ret i8 [[RES]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i8 [ 0, %entry ], [ %iv.next, %loop.latch ] + %s = shl nuw i8 1, %iv + %a = and i8 %s, %g + %c = icmp eq i8 %a, 0 + br i1 %c, label %loop.latch, label %return + +loop.latch: + %iv.next = add nuw nsw i8 %iv, 1 + %ec = icmp eq i8 %iv.next, 4 + br i1 %ec, label %return, label %loop.header + +return: + %res.iv1 = phi i8 [ 32, %loop.latch ], [ %iv, %loop.header ] + %res.iv2 = phi i8 [ 0, %loop.latch ], [ %iv, %loop.header ] + %res = add i8 %res.iv1, %res.iv2 + ret i8 %res +} + +define i32 @iv_used_in_exit_with_loads(ptr align 4 dereferenceable(128) %src) { +; CHECK-LABEL: define i32 @iv_used_in_exit_with_loads( +; CHECK-SAME: ptr align 4 dereferenceable(128) [[SRC:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 [[IV]] +; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4 +; CHECK-NEXT: [[C:%.*]] = icmp eq i32 [[L]], 0 +; CHECK-NEXT: br i1 [[C]], label %[[LOOP_LATCH]], label %[[RETURN:.*]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], 32 +; CHECK-NEXT: br i1 [[EC]], label %[[RETURN]], label %[[LOOP_HEADER]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[RES_IV1:%.*]] = phi i32 [ 32, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ] +; CHECK-NEXT: [[RES_IV2:%.*]] = phi i32 [ 0, %[[LOOP_LATCH]] ], [ [[IV]], %[[LOOP_HEADER]] ] +; CHECK-NEXT: [[RES:%.*]] = add i32 [[RES_IV1]], [[RES_IV2]] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep = getelementptr inbounds i32, ptr %src, i32 %iv + %l = load i32, ptr %gep + %c = icmp eq i32 %l, 0 + br i1 %c, label %loop.latch, label %return + +loop.latch: + %iv.next = add nuw nsw i32 %iv, 1 + %ec = icmp eq i32 %iv.next, 32 + br i1 %ec, label %return, label %loop.header + +return: + %res.iv1 = phi i32 [ 32, %loop.latch ], [ %iv, %loop.header ] + %res.iv2 = phi i32 [ 0, %loop.latch ], [ %iv, %loop.header ] + %res = add i32 %res.iv1, %res.iv2 + ret i32 %res +} diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll index 0f99ed576f1fe..01f65546e3534 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-interleave.ll @@ -18,7 +18,8 @@ define i64 @multi_exiting_to_different_exits_live_in_exit_values() { ; VF4IC4-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i32>, ptr [[TMP1]], align 4 ; VF4IC4-NEXT: [[TMP8:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD3]], splat (i32 10) ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]]) +; VF4IC4-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP8]] +; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) ; VF4IC4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 ; VF4IC4-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; VF4IC4-NEXT: br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -95,7 +96,8 @@ define i64 @same_exit_block_pre_inc_use1() { ; VF4IC4-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1 ; VF4IC4-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP13]] +; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]]) ; VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 ; VF4IC4-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; VF4IC4-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -170,7 +172,8 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() { ; VF4IC4-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1 ; VF4IC4-NEXT: [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], splat (i8 72) ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]]) +; VF4IC4-NEXT: [[TMP9:%.*]] = freeze <4 x i1> [[TMP17]] +; VF4IC4-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP9]]) ; VF4IC4-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; VF4IC4-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; VF4IC4-NEXT: br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -242,7 +245,8 @@ define i64 @same_exit_block_post_inc_use() { ; VF4IC4-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1 ; VF4IC4-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP13]] +; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]]) ; VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 ; VF4IC4-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; VF4IC4-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -321,7 +325,8 @@ define i64 @diff_exit_block_pre_inc_use1() { ; VF4IC4-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1 ; VF4IC4-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP13]] +; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]]) ; VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 ; VF4IC4-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; VF4IC4-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -407,7 +412,8 @@ define i64 @diff_exit_block_post_inc_use1() { ; VF4IC4-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1 ; VF4IC4-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; VF4IC4-NEXT: [[TMP11:%.*]] = freeze <4 x i1> [[TMP13]] +; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]]) ; VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 64 ; VF4IC4-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; VF4IC4-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -497,7 +503,8 @@ define i64 @same_exit_block_pre_inc_use1_reverse() { ; VF4IC4-NEXT: [[REVERSE15:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD14]], <4 x i8> poison, <4 x i32> ; VF4IC4-NEXT: [[TMP21:%.*]] = icmp ne <4 x i8> [[REVERSE14]], [[REVERSE15]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]]) +; VF4IC4-NEXT: [[TMP13:%.*]] = freeze <4 x i1> [[TMP21]] +; VF4IC4-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) ; VF4IC4-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1020 ; VF4IC4-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; VF4IC4-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -576,7 +583,8 @@ define i8 @same_exit_block_use_loaded_value() { ; VF4IC4-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i8>, ptr [[TMP19]], align 1 ; VF4IC4-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD3]], [[WIDE_LOAD7]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; VF4IC4-NEXT: [[TMP8:%.*]] = freeze <4 x i1> [[TMP13]] +; VF4IC4-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]]) ; VF4IC4-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; VF4IC4-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; VF4IC4-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -659,7 +667,8 @@ define i8 @same_exit_block_reverse_use_loaded_value() { ; VF4IC4-NEXT: [[REVERSE14:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD13]], <4 x i8> poison, <4 x i32> ; VF4IC4-NEXT: [[TMP21:%.*]] = icmp ne <4 x i8> [[REVERSE6]], [[REVERSE14]] ; VF4IC4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; VF4IC4-NEXT: [[TMP25:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP21]]) +; VF4IC4-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP21]] +; VF4IC4-NEXT: [[TMP25:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]]) ; VF4IC4-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1020 ; VF4IC4-NEXT: [[TMP27:%.*]] = or i1 [[TMP25]], [[TMP26]] ; VF4IC4-NEXT: br i1 [[TMP27]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll index 38553b2385b06..ddf238ffba4b4 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll @@ -25,7 +25,8 @@ define i64 @same_exit_block_phi_of_consts() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -102,7 +103,8 @@ define i64 @diff_exit_block_phi_of_consts() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -200,7 +202,8 @@ define i32 @diff_exit_block_needs_scev_check(i32 %end) { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i32>, ptr [[TMP13]], align 4 ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP14]]) +; CHECK-NEXT: [[TMP20:%.*]] = freeze <4 x i1> [[TMP14]] +; CHECK-NEXT: [[TMP15:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP20]]) ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP17:%.*]] = or i1 [[TMP15]], [[TMP16]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -283,7 +286,8 @@ define i32 @diff_blocks_invariant_early_exit_cond(ptr %s) { ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[COND]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[TMP0:%.*]] = xor <4 x i1> [[BROADCAST_SPLAT]], splat (i1 true) -; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP0]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP0]] +; CHECK-NEXT: [[TMP1:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -359,7 +363,8 @@ define void @inner_loop_trip_count_depends_on_outer_iv(ptr align 8 dereferenceab ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x ptr>, ptr [[TMP2]], align 8 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x ptr> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]]) +; CHECK-NEXT: [[TMP6:%.*]] = freeze <4 x i1> [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] ; CHECK-NEXT: br i1 [[TMP5]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -456,7 +461,8 @@ define i64 @loop_guard_needed_to_prove_dereferenceable(i32 %x, i1 %cmp2) { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP3]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) +; CHECK-NEXT: [[TMP10:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP10]]) ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] ; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll index a21666a31b6a2..6050119b67ff4 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit_live_outs.ll @@ -24,7 +24,8 @@ define i64 @same_exit_block_pre_inc_use1() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -104,7 +105,8 @@ define i32 @same_exit_block_pre_inc_use1_iv64_endi32_step2() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -187,7 +189,8 @@ define i32 @same_exit_block_pre_inc_use1_iv128_endi32_step2() { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP2]], align 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], splat (i8 3) ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i128 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]]) ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i128 [[INDEX_NEXT2]], 64 ; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -272,7 +275,8 @@ define float @same_exit_block_pre_inc_use1_iv64_endf32() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -360,7 +364,8 @@ define ptr @same_exit_block_pre_inc_use1_iv64_endptr() { ; CHECK-NEXT: [[WIDE_LOAD6:%.*]] = load <4 x i8>, ptr [[TMP13]], align 1 ; CHECK-NEXT: [[TMP15:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD6]] ; CHECK-NEXT: [[INDEX_NEXT7]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP15]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP15]] +; CHECK-NEXT: [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT7]], 64 ; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]] ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -442,7 +447,8 @@ define ptr @same_exit_block_pre_inc_use1_ivptr() { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1 ; CHECK-NEXT: [[TMP11:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], splat (i8 72) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP11]]) +; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i1> [[TMP11]] +; CHECK-NEXT: [[TMP12:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]]) ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]] ; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -518,7 +524,8 @@ define i64 @same_exit_block_pre_inc1_use_inv_cond(i1 %cond) { ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[COND]], <4 x i1> [[TMP5]], <4 x i1> zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = xor <4 x i1> [[TMP6]], splat (i1 true) -; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]]) +; CHECK-NEXT: [[TMP12:%.*]] = freeze <4 x i1> [[TMP7]] +; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP12]]) ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -600,7 +607,8 @@ define i64 @same_exit_block_pre_inc_use1_gep_two_indices() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -680,7 +688,8 @@ define i64 @same_exit_block_pre_inc_use1_alloca_diff_type() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] @@ -760,7 +769,8 @@ define i64 @same_exit_block_pre_inc_use2() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] @@ -837,7 +847,8 @@ define i64 @same_exit_block_pre_inc_use3() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] @@ -916,7 +927,8 @@ define i64 @same_exit_block_pre_inc_use4() { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP2]], align 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp uge <4 x i64> [[VEC_IND]], [[WIDE_LOAD]] ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) +; CHECK-NEXT: [[TMP2:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP5:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP2]]) ; CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64 ; CHECK-NEXT: [[TMP7:%.*]] = or i1 [[TMP5]], [[TMP6]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add <4 x i64> [[VEC_IND]], splat (i64 4) @@ -993,7 +1005,8 @@ define i64 @same_exit_block_post_inc_use() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1 ; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP13]] +; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]] ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP26:![0-9]+]] @@ -1069,7 +1082,8 @@ define ptr @same_exit_block_post_inc_use1_ivptr() { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP5]], align 1 ; CHECK-NEXT: [[TMP15:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], splat (i8 72) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP15]]) +; CHECK-NEXT: [[TMP1:%.*]] = freeze <4 x i1> [[TMP15]] +; CHECK-NEXT: [[TMP16:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP1]]) ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: [[TMP18:%.*]] = or i1 [[TMP16]], [[TMP17]] ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP28:![0-9]+]] @@ -1143,7 +1157,8 @@ define i64 @same_exit_block_post_inc_use2() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1 ; CHECK-NEXT: [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP17]] +; CHECK-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP30:![0-9]+]] @@ -1224,7 +1239,8 @@ define i64 @diff_exit_block_pre_inc_use1() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP32:![0-9]+]] @@ -1311,7 +1327,8 @@ define i64 @diff_exit_block_pre_inc_use2() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP34:![0-9]+]] @@ -1395,7 +1412,8 @@ define i64 @diff_exit_block_pre_inc_use3() { ; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]] ; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX2]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] @@ -1480,7 +1498,8 @@ define i64 @diff_exit_block_post_inc_use1() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1 ; CHECK-NEXT: [[TMP13:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP13]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP13]] +; CHECK-NEXT: [[TMP14:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP16:%.*]] = or i1 [[TMP14]], [[TMP15]] ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] @@ -1567,7 +1586,8 @@ define i64 @diff_exit_block_post_inc_use2() { ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP15]], align 1 ; CHECK-NEXT: [[TMP17:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP17]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP17]] +; CHECK-NEXT: [[TMP18:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP20:%.*]] = or i1 [[TMP18]], [[TMP19]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] @@ -1656,7 +1676,8 @@ define i64 @diff_exit_block_post_inc_use3(i64 %start) { ; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load <4 x i8>, ptr [[TMP17]], align 1 ; CHECK-NEXT: [[TMP19:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD3]] ; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP19]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP19]] +; CHECK-NEXT: [[TMP20:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 64 ; CHECK-NEXT: [[TMP22:%.*]] = or i1 [[TMP20]], [[TMP21]] ; CHECK-NEXT: br i1 [[TMP22]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] @@ -1748,7 +1769,8 @@ define i64 @loop_contains_safe_call() { ; CHECK-NEXT: [[TMP3:%.*]] = call fast <4 x float> @llvm.sqrt.v4f32(<4 x float> [[WIDE_LOAD]]) ; CHECK-NEXT: [[TMP5:%.*]] = fcmp fast oge <4 x float> [[TMP3]], splat (float 3.000000e+00) ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64 ; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP44:![0-9]+]] @@ -1824,7 +1846,8 @@ define i64 @loop_contains_safe_div() { ; CHECK-NEXT: [[TMP3:%.*]] = udiv <4 x i32> [[WIDE_LOAD]], splat (i32 20000) ; CHECK-NEXT: [[TMP5:%.*]] = icmp ne <4 x i32> [[TMP3]], splat (i32 1) ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP5]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT2]], 64 ; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP46:![0-9]+]] @@ -1901,7 +1924,8 @@ define i64 @loop_contains_load_after_early_exit(ptr dereferenceable(1024) align( ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[TMP4]], i32 0 ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i64>, ptr [[TMP5]], align 8 ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP48:![0-9]+]] @@ -1984,7 +2008,8 @@ define i64 @same_exit_block_pre_inc_use1_reverse() { ; CHECK-NEXT: [[REVERSE3:%.*]] = shufflevector <4 x i8> [[WIDE_LOAD2]], <4 x i8> poison, <4 x i32> ; CHECK-NEXT: [[TMP8:%.*]] = icmp ne <4 x i8> [[REVERSE]], [[REVERSE3]] ; CHECK-NEXT: [[INDEX_NEXT4]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP8]]) +; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]]) ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT4]], 1020 ; CHECK-NEXT: [[TMP11:%.*]] = or i1 [[TMP9]], [[TMP10]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP50:![0-9]+]] @@ -2111,7 +2136,8 @@ define i64 @same_exit_block_pre_inc_use1_deref_ptrs(ptr dereferenceable(1024) %p ; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP4]], align 1 ; CHECK-NEXT: [[TMP6:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP6]]) +; CHECK-NEXT: [[TMP4:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP7:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP4]]) ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 64 ; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP52:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll index fdd5e0e7958ec..69d0076497c66 100644 --- a/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll +++ b/llvm/test/Transforms/LoopVectorize/vector-loop-backedge-elimination-early-exit.ll @@ -21,7 +21,8 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn ; VF8UF1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1 ; VF8UF1-NEXT: [[TMP3:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer ; VF8UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 -; VF8UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]]) +; VF8UF1-NEXT: [[TMP7:%.*]] = freeze <8 x i1> [[TMP3]] +; VF8UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP7]]) ; VF8UF1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 ; VF8UF1-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] ; VF8UF1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -55,18 +56,19 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn ; VF8UF2: [[VECTOR_PH]]: ; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]] ; VF8UF2: [[VECTOR_BODY]]: -; VF8UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; VF8UF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]] -; VF8UF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 -; VF8UF2-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 -; VF8UF2-NEXT: [[TMP6:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD1]], zeroinitializer -; VF8UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 -; VF8UF2-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP6]]) -; VF8UF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 -; VF8UF2-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] -; VF8UF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; VF8UF2-NEXT: [[IV1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF8UF2-NEXT: [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]] +; VF8UF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P_SRC1]], i32 0 +; VF8UF2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; VF8UF2-NEXT: [[TMP2:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer +; VF8UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[IV1]], 8 +; VF8UF2-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP2]] +; VF8UF2-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]]) +; VF8UF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 +; VF8UF2-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] +; VF8UF2-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; VF8UF2: [[MIDDLE_SPLIT]]: -; VF8UF2-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF8UF2-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] ; VF8UF2: [[MIDDLE_BLOCK]]: ; VF8UF2-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; VF8UF2: [[VECTOR_EARLY_EXIT]]: @@ -75,13 +77,13 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn ; VF8UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] ; VF8UF2-NEXT: br label %[[LOOP_HEADER:.*]] ; VF8UF2: [[LOOP_HEADER]]: -; VF8UF2-NEXT: [[IV1:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] -; VF8UF2-NEXT: [[P_SRC1:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV1]] -; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC1]], align 1 +; VF8UF2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; VF8UF2-NEXT: [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]] +; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 1 ; VF8UF2-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0 ; VF8UF2-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]] ; VF8UF2: [[LOOP_LATCH]]: -; VF8UF2-NEXT: [[IV_NEXT]] = add nsw i64 [[IV1]], 1 +; VF8UF2-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1 ; VF8UF2-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 16 ; VF8UF2-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]] ; VF8UF2: [[EXIT]]: @@ -98,7 +100,8 @@ define i8 @test_early_exit_max_tc_less_than_16(ptr dereferenceable(16) %A) nosyn ; VF16UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 0 ; VF16UF1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1 ; VF16UF1-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], zeroinitializer -; VF16UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; VF16UF1-NEXT: [[TMP1:%.*]] = freeze <16 x i1> [[TMP3]] +; VF16UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]]) ; VF16UF1-NEXT: br label %[[MIDDLE_SPLIT:.*]] ; VF16UF1: [[MIDDLE_SPLIT]]: ; VF16UF1-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] @@ -157,7 +160,8 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer ; VF8UF1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP2]], align 1 ; VF8UF1-NEXT: [[TMP3:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer ; VF8UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 -; VF8UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]]) +; VF8UF1-NEXT: [[TMP7:%.*]] = freeze <8 x i1> [[TMP3]] +; VF8UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP7]]) ; VF8UF1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 ; VF8UF1-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] ; VF8UF1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -196,15 +200,16 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer ; VF8UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; VF8UF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]] ; VF8UF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 -; VF8UF2-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 -; VF8UF2-NEXT: [[TMP6:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD1]], zeroinitializer +; VF8UF2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; VF8UF2-NEXT: [[TMP6:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer ; VF8UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 -; VF8UF2-NEXT: [[TMP3:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP6]]) -; VF8UF2-NEXT: [[TMP4:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 -; VF8UF2-NEXT: [[TMP5:%.*]] = or i1 [[TMP3]], [[TMP4]] -; VF8UF2-NEXT: br i1 [[TMP5]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; VF8UF2-NEXT: [[TMP3:%.*]] = freeze <8 x i1> [[TMP6]] +; VF8UF2-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP3]]) +; VF8UF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 +; VF8UF2-NEXT: [[TMP9:%.*]] = or i1 [[TMP4]], [[TMP5]] +; VF8UF2-NEXT: br i1 [[TMP9]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; VF8UF2: [[MIDDLE_SPLIT]]: -; VF8UF2-NEXT: br i1 [[TMP3]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF8UF2-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] ; VF8UF2: [[MIDDLE_BLOCK]]: ; VF8UF2-NEXT: br i1 true, label %[[EXIT:.*]], label %[[SCALAR_PH]] ; VF8UF2: [[VECTOR_EARLY_EXIT]]: @@ -238,7 +243,8 @@ define i64 @test_early_exit_max_tc_less_than_16_with_iv_used_outside(ptr derefer ; VF16UF1-NEXT: [[TMP2:%.*]] = getelementptr inbounds i8, ptr [[A]], i32 0 ; VF16UF1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP2]], align 1 ; VF16UF1-NEXT: [[TMP3:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], zeroinitializer -; VF16UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; VF16UF1-NEXT: [[TMP1:%.*]] = freeze <16 x i1> [[TMP3]] +; VF16UF1-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP1]]) ; VF16UF1-NEXT: br label %[[MIDDLE_SPLIT:.*]] ; VF16UF1: [[MIDDLE_SPLIT]]: ; VF16UF1-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] @@ -284,3 +290,147 @@ exit: %res = phi i64 [ %iv, %loop.header ], [ 1, %loop.latch ] ret i64 %res } + +define i8 @test_early_exit_max_vector_tc_eq_16(ptr dereferenceable(17) %A) nosync nofree { +; VF8UF1-LABEL: define i8 @test_early_exit_max_vector_tc_eq_16( +; VF8UF1-SAME: ptr dereferenceable(17) [[A:%.*]]) #[[ATTR0]] { +; VF8UF1-NEXT: [[ENTRY:.*]]: +; VF8UF1-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; VF8UF1: [[VECTOR_PH]]: +; VF8UF1-NEXT: br label %[[VECTOR_BODY:.*]] +; VF8UF1: [[VECTOR_BODY]]: +; VF8UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF8UF1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]] +; VF8UF1-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 +; VF8UF1-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[TMP6]], align 1 +; VF8UF1-NEXT: [[TMP1:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD]], zeroinitializer +; VF8UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 +; VF8UF1-NEXT: [[TMP5:%.*]] = freeze <8 x i1> [[TMP1]] +; VF8UF1-NEXT: [[TMP2:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP5]]) +; VF8UF1-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 +; VF8UF1-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]] +; VF8UF1-NEXT: br i1 [[TMP4]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; VF8UF1: [[MIDDLE_SPLIT]]: +; VF8UF1-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF8UF1: [[MIDDLE_BLOCK]]: +; VF8UF1-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; VF8UF1: [[VECTOR_EARLY_EXIT]]: +; VF8UF1-NEXT: br label %[[EXIT]] +; VF8UF1: [[SCALAR_PH]]: +; VF8UF1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; VF8UF1-NEXT: br label %[[LOOP_HEADER:.*]] +; VF8UF1: [[LOOP_HEADER]]: +; VF8UF1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; VF8UF1-NEXT: [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]] +; VF8UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 1 +; VF8UF1-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0 +; VF8UF1-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]] +; VF8UF1: [[LOOP_LATCH]]: +; VF8UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1 +; VF8UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 17 +; VF8UF1-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]] +; VF8UF1: [[EXIT]]: +; VF8UF1-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ] +; VF8UF1-NEXT: ret i8 [[RES]] +; +; VF8UF2-LABEL: define i8 @test_early_exit_max_vector_tc_eq_16( +; VF8UF2-SAME: ptr dereferenceable(17) [[A:%.*]]) #[[ATTR0]] { +; VF8UF2-NEXT: [[ENTRY:.*]]: +; VF8UF2-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; VF8UF2: [[VECTOR_PH]]: +; VF8UF2-NEXT: br label %[[VECTOR_BODY:.*]] +; VF8UF2: [[VECTOR_BODY]]: +; VF8UF2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF8UF2-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]] +; VF8UF2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 +; VF8UF2-NEXT: [[WIDE_LOAD1:%.*]] = load <8 x i8>, ptr [[TMP1]], align 1 +; VF8UF2-NEXT: [[TMP3:%.*]] = icmp eq <8 x i8> [[WIDE_LOAD1]], zeroinitializer +; VF8UF2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 +; VF8UF2-NEXT: [[TMP6:%.*]] = freeze <8 x i1> [[TMP3]] +; VF8UF2-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v8i1(<8 x i1> [[TMP6]]) +; VF8UF2-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 +; VF8UF2-NEXT: [[TMP7:%.*]] = or i1 [[TMP4]], [[TMP5]] +; VF8UF2-NEXT: br i1 [[TMP7]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; VF8UF2: [[MIDDLE_SPLIT]]: +; VF8UF2-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF8UF2: [[MIDDLE_BLOCK]]: +; VF8UF2-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; VF8UF2: [[VECTOR_EARLY_EXIT]]: +; VF8UF2-NEXT: br label %[[EXIT]] +; VF8UF2: [[SCALAR_PH]]: +; VF8UF2-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; VF8UF2-NEXT: br label %[[LOOP_HEADER:.*]] +; VF8UF2: [[LOOP_HEADER]]: +; VF8UF2-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; VF8UF2-NEXT: [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]] +; VF8UF2-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 1 +; VF8UF2-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0 +; VF8UF2-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]] +; VF8UF2: [[LOOP_LATCH]]: +; VF8UF2-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1 +; VF8UF2-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 17 +; VF8UF2-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP7:![0-9]+]] +; VF8UF2: [[EXIT]]: +; VF8UF2-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ] +; VF8UF2-NEXT: ret i8 [[RES]] +; +; VF16UF1-LABEL: define i8 @test_early_exit_max_vector_tc_eq_16( +; VF16UF1-SAME: ptr dereferenceable(17) [[A:%.*]]) #[[ATTR0]] { +; VF16UF1-NEXT: [[ENTRY:.*]]: +; VF16UF1-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; VF16UF1: [[VECTOR_PH]]: +; VF16UF1-NEXT: br label %[[VECTOR_BODY:.*]] +; VF16UF1: [[VECTOR_BODY]]: +; VF16UF1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; VF16UF1-NEXT: [[TMP0:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[INDEX]] +; VF16UF1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i8, ptr [[TMP0]], i32 0 +; VF16UF1-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[TMP4]], align 1 +; VF16UF1-NEXT: [[TMP1:%.*]] = icmp eq <16 x i8> [[WIDE_LOAD]], zeroinitializer +; VF16UF1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16 +; VF16UF1-NEXT: [[TMP3:%.*]] = freeze <16 x i1> [[TMP1]] +; VF16UF1-NEXT: [[TMP2:%.*]] = call i1 @llvm.vector.reduce.or.v16i1(<16 x i1> [[TMP3]]) +; VF16UF1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 16 +; VF16UF1-NEXT: [[TMP6:%.*]] = or i1 [[TMP2]], [[TMP5]] +; VF16UF1-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; VF16UF1: [[MIDDLE_SPLIT]]: +; VF16UF1-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; VF16UF1: [[MIDDLE_BLOCK]]: +; VF16UF1-NEXT: br i1 false, label %[[EXIT:.*]], label %[[SCALAR_PH]] +; VF16UF1: [[VECTOR_EARLY_EXIT]]: +; VF16UF1-NEXT: br label %[[EXIT]] +; VF16UF1: [[SCALAR_PH]]: +; VF16UF1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 16, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; VF16UF1-NEXT: br label %[[LOOP_HEADER:.*]] +; VF16UF1: [[LOOP_HEADER]]: +; VF16UF1-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; VF16UF1-NEXT: [[P_SRC:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[IV]] +; VF16UF1-NEXT: [[L:%.*]] = load i8, ptr [[P_SRC]], align 1 +; VF16UF1-NEXT: [[C:%.*]] = icmp eq i8 [[L]], 0 +; VF16UF1-NEXT: br i1 [[C]], label %[[EXIT]], label %[[LOOP_LATCH]] +; VF16UF1: [[LOOP_LATCH]]: +; VF16UF1-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1 +; VF16UF1-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV_NEXT]], 17 +; VF16UF1-NEXT: br i1 [[CMP]], label %[[EXIT]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP5:![0-9]+]] +; VF16UF1: [[EXIT]]: +; VF16UF1-NEXT: [[RES:%.*]] = phi i8 [ 0, %[[LOOP_HEADER]] ], [ 1, %[[LOOP_LATCH]] ], [ 1, %[[MIDDLE_BLOCK]] ], [ 0, %[[VECTOR_EARLY_EXIT]] ] +; VF16UF1-NEXT: ret i8 [[RES]] +; +entry: + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %p.src = getelementptr inbounds i8, ptr %A, i64 %iv + %l = load i8, ptr %p.src, align 1 + %c = icmp eq i8 %l, 0 + br i1 %c, label %exit, label %loop.latch + +loop.latch: + %iv.next = add nsw i64 %iv, 1 + %cmp = icmp eq i64 %iv.next, 17 + br i1 %cmp, label %exit, label %loop.header + +exit: + %res = phi i8 [ 0, %loop.header ], [ 1, %loop.latch ] + ret i8 %res +} diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll new file mode 100644 index 0000000000000..a3b8736a06ec7 --- /dev/null +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll @@ -0,0 +1,137 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -O3 -S %s | FileCheck %s + +target triple = "arm64-apple-macosx15.0.0" + +define i64 @std_find_i16_constant_offset_with_assumptions(ptr %first.coerce, i16 noundef signext %s) nofree nosync { +; CHECK-LABEL: define i64 @std_find_i16_constant_offset_with_assumptions( +; CHECK-SAME: ptr [[FIRST_COERCE:%.*]], i16 noundef signext [[S:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[FIRST_COERCE]], i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[FIRST_COERCE]], i64 256) ] +; CHECK-NEXT: [[COERCE_VAL_IP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 256 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <8 x i16> poison, i16 [[S]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <8 x i16> [[BROADCAST_SPLATINSERT]], <8 x i16> poison, <8 x i32> zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2 +; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <8 x i16> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD_FR]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 +; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i1> [[TMP0]] to i8 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i8 [[TMP1]], 0 +; CHECK-NEXT: [[TMP3:%.*]] = icmp eq i64 [[INDEX_NEXT]], 128 +; CHECK-NEXT: [[TMP4:%.*]] = or i1 [[TMP2]], [[TMP3]] +; CHECK-NEXT: br i1 [[TMP4]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_SPLIT]]: +; CHECK-NEXT: br i1 [[TMP2]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[RETURN:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: [[TMP5:%.*]] = tail call i64 @llvm.experimental.cttz.elts.i64.v8i1(<8 x i1> [[TMP0]], i1 true) +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 [[TMP7]] +; CHECK-NEXT: br label %[[RETURN]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[__FIRST_ADDR_0_LCSSA_I_I_PH:%.*]] = phi ptr [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ], [ [[COERCE_VAL_IP]], %[[MIDDLE_SPLIT]] ] +; CHECK-NEXT: [[DOTPRE:%.*]] = ptrtoint ptr [[__FIRST_ADDR_0_LCSSA_I_I_PH]] to i64 +; CHECK-NEXT: ret i64 [[DOTPRE]] +; +entry: + %first = alloca { ptr }, align 8 + %s.addr = alloca i16, align 2 + store ptr %first.coerce, ptr %first, align 8 + store i16 %s, ptr %s.addr, align 2 + %0 = load ptr, ptr %first, align 8 + call void @llvm.assume(i1 true) [ "align"(ptr %0, i64 2) ] + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %0, i64 256) ] + %start.ptr = load ptr, ptr %first, align 8 + %1 = load i64, ptr %first, align 8 + %coerce.val.pi.i = add i64 %1, 256 + %coerce.val.ip = inttoptr i64 %coerce.val.pi.i to ptr + %cmp.not6.i.i = icmp eq ptr %start.ptr, %coerce.val.ip + br i1 %cmp.not6.i.i, label %return, label %loop.ph + +loop.ph: + %2 = load i16, ptr %s.addr, align 2 + br label %loop.header + +loop.header: + %ptr.iv = phi ptr [ %start.ptr, %loop.ph ], [ %ptr.iv.next, %loop.latch ] + %3 = load i16, ptr %ptr.iv, align 2 + %cmp2.i.i = icmp eq i16 %3, %2 + br i1 %cmp2.i.i, label %return, label %loop.latch + +loop.latch: + %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 2 + %cmp.not.i.i = icmp eq ptr %ptr.iv.next, %coerce.val.ip + br i1 %cmp.not.i.i, label %return, label %loop.header + +return: + %merge = phi ptr [ %start.ptr, %entry ], [ %coerce.val.ip, %loop.latch ], [ %ptr.iv, %loop.header ] + %res = ptrtoint ptr %merge to i64 + ret i64 %res +} + +define i64 @std_find_i16_constant_offset_no_assumptions(ptr %first.coerce, i16 noundef signext %s) nofree nosync { +; CHECK-LABEL: define i64 @std_find_i16_constant_offset_no_assumptions( +; CHECK-SAME: ptr [[FIRST_COERCE:%.*]], i16 noundef signext [[S:%.*]]) local_unnamed_addr #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[COERCE_VAL_IP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 256 +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[FIRST_COERCE]], %[[ENTRY]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[TMP1:%.*]] = load i16, ptr [[PTR_IV]], align 2 +; CHECK-NEXT: [[CMP2_I_I:%.*]] = icmp eq i16 [[TMP1]], [[S]] +; CHECK-NEXT: br i1 [[CMP2_I_I]], label %[[RETURN:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[PTR_IV]], i64 2 +; CHECK-NEXT: [[CMP_NOT_I_I:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[COERCE_VAL_IP]] +; CHECK-NEXT: br i1 [[CMP_NOT_I_I]], label %[[RETURN]], label %[[LOOP_HEADER]] +; CHECK: [[RETURN]]: +; CHECK-NEXT: [[MERGE_PH:%.*]] = phi ptr [ [[COERCE_VAL_IP]], %[[LOOP_LATCH]] ], [ [[PTR_IV]], %[[LOOP_HEADER]] ] +; CHECK-NEXT: [[DOTPRE:%.*]] = ptrtoint ptr [[MERGE_PH]] to i64 +; CHECK-NEXT: ret i64 [[DOTPRE]] +; +entry: + %first = alloca { ptr }, align 8 + %s.addr = alloca i16, align 2 + store ptr %first.coerce, ptr %first, align 8 + store i16 %s, ptr %s.addr, align 2 + %0 = load ptr, ptr %first, align 8 + %start.ptr = load ptr, ptr %first, align 8 + %1 = load i64, ptr %first, align 8 + %coerce.val.pi.i = add i64 %1, 256 + %coerce.val.ip = inttoptr i64 %coerce.val.pi.i to ptr + %cmp.not6.i.i = icmp eq ptr %start.ptr, %coerce.val.ip + br i1 %cmp.not6.i.i, label %return, label %loop.ph + +loop.ph: + %2 = load i16, ptr %s.addr, align 2 + br label %loop.header + +loop.header: + %ptr.iv = phi ptr [ %start.ptr, %loop.ph ], [ %ptr.iv.next, %loop.latch ] + %3 = load i16, ptr %ptr.iv, align 2 + %cmp2.i.i = icmp eq i16 %3, %2 + br i1 %cmp2.i.i, label %return, label %loop.latch + +loop.latch: + %ptr.iv.next = getelementptr inbounds nuw i8, ptr %ptr.iv, i64 2 + %cmp.not.i.i = icmp eq ptr %ptr.iv.next, %coerce.val.ip + br i1 %cmp.not.i.i, label %return, label %loop.header + +return: + %merge = phi ptr [ %start.ptr, %entry ], [ %coerce.val.ip, %loop.latch ], [ %ptr.iv, %loop.header ] + %res = ptrtoint ptr %merge to i64 + ret i64 %res +} + +declare void @llvm.assume(i1 noundef) +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +;. From f6c35e283f2339d0948c05884a84703fcf520d31 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Sep 2025 16:21:57 +0100 Subject: [PATCH 26/60] [SCEV] Add tests for folding multiply/divide by constants. Add extra test coverage for follow-up to https://github.com/llvm/llvm-project/pull/156730. (cherry picked from commit 45a22142fc77aabb10d056edc87ab8841a18e975) --- .../ScalarEvolution/mul-udiv-folds.ll | 187 ++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll diff --git a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll new file mode 100644 index 0000000000000..9f4360d2ae383 --- /dev/null +++ b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll @@ -0,0 +1,187 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes='print' -disable-output %s 2>&1 | FileCheck %s + +declare void @use(ptr) + +define void @udiv4_and_udiv2_mul_4(i1 %c, ptr %A) { +; CHECK-LABEL: 'udiv4_and_udiv2_mul_4' +; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2_mul_4 +; CHECK-NEXT: %start = select i1 %c, i32 512, i32 0 +; CHECK-NEXT: --> %start U: [0,513) S: [0,513) +; CHECK-NEXT: %div.2 = lshr i32 %start, 1 +; CHECK-NEXT: --> (%start /u 2) U: [0,257) S: [0,257) +; CHECK-NEXT: %div.4 = lshr i32 %start, 2 +; CHECK-NEXT: --> (%start /u 4) U: [0,129) S: [0,129) +; CHECK-NEXT: %iv.start = zext i32 %div.4 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 4) U: [0,129) S: [0,129) +; CHECK-NEXT: %wide.trip.count = zext i32 %div.2 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 2) U: [0,257) S: [0,257) +; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {((zext i32 %start to i64) /u 4),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 2) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep = getelementptr i32, ptr %A, i64 %iv +; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + (4 * ((zext i32 %start to i64) /u 2)) + (-4 * ((zext i32 %start to i64) /u 4)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = add i64 %iv, 1 +; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 4)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 2)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2_mul_4 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %start = select i1 %c, i32 512, i32 0 + %div.2 = lshr i32 %start, 1 + %div.4 = lshr i32 %start, 2 + %iv.start = zext i32 %div.4 to i64 + %wide.trip.count = zext i32 %div.2 to i64 + br label %loop + +loop: + %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] + %gep = getelementptr i32, ptr %A, i64 %iv + call void @use(ptr %gep) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, %wide.trip.count + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @udiv4_and_udiv2_mul_1(i1 %c, ptr %A) { +; CHECK-LABEL: 'udiv4_and_udiv2_mul_1' +; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2_mul_1 +; CHECK-NEXT: %start = select i1 %c, i32 512, i32 0 +; CHECK-NEXT: --> %start U: [0,513) S: [0,513) +; CHECK-NEXT: %div.2 = lshr i32 %start, 1 +; CHECK-NEXT: --> (%start /u 2) U: [0,257) S: [0,257) +; CHECK-NEXT: %div.4 = lshr i32 %start, 2 +; CHECK-NEXT: --> (%start /u 4) U: [0,129) S: [0,129) +; CHECK-NEXT: %iv.start = zext i32 %div.4 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 4) U: [0,129) S: [0,129) +; CHECK-NEXT: %wide.trip.count = zext i32 %div.2 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 2) U: [0,257) S: [0,257) +; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {((zext i32 %start to i64) /u 4),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 2) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep = getelementptr i8, ptr %A, i64 %iv +; CHECK-NEXT: --> {(((zext i32 %start to i64) /u 4) + %A),+,1}<%loop> U: full-set S: full-set Exits: (((zext i32 %start to i64) /u 2) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = add i64 %iv, 1 +; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 4)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 2)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2_mul_1 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %start = select i1 %c, i32 512, i32 0 + %div.2 = lshr i32 %start, 1 + %div.4 = lshr i32 %start, 2 + %iv.start = zext i32 %div.4 to i64 + %wide.trip.count = zext i32 %div.2 to i64 + br label %loop + +loop: + %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] + %gep = getelementptr i8, ptr %A, i64 %iv + call void @use(ptr %gep) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, %wide.trip.count + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + + +define void @udiv3_and_udiv5_mul_4(i1 %c, ptr %A) { +; CHECK-LABEL: 'udiv3_and_udiv5_mul_4' +; CHECK-NEXT: Classifying expressions for: @udiv3_and_udiv5_mul_4 +; CHECK-NEXT: %start = select i1 %c, i32 512, i32 0 +; CHECK-NEXT: --> %start U: [0,513) S: [0,513) +; CHECK-NEXT: %div.3 = udiv i32 %start, 3 +; CHECK-NEXT: --> (%start /u 3) U: [0,171) S: [0,171) +; CHECK-NEXT: %div.5 = udiv i32 %start, 5 +; CHECK-NEXT: --> (%start /u 5) U: [0,103) S: [0,103) +; CHECK-NEXT: %iv.start = zext i32 %div.5 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 5) U: [0,103) S: [0,103) +; CHECK-NEXT: %wide.trip.count = zext i32 %div.3 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 3) U: [0,171) S: [0,171) +; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {((zext i32 %start to i64) /u 5),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 3) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep = getelementptr i8, ptr %A, i64 %iv +; CHECK-NEXT: --> {(((zext i32 %start to i64) /u 5) + %A),+,1}<%loop> U: full-set S: full-set Exits: (((zext i32 %start to i64) /u 3) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = add i64 %iv, 1 +; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 5)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 3)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @udiv3_and_udiv5_mul_4 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 5)) + ((zext i32 %start to i64) /u 3)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 5)) + ((zext i32 %start to i64) /u 3)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %start = select i1 %c, i32 512, i32 0 + %div.3 = udiv i32 %start, 3 + %div.5 = udiv i32 %start, 5 + %iv.start = zext i32 %div.5 to i64 + %wide.trip.count = zext i32 %div.3 to i64 + br label %loop + +loop: + %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] + %gep = getelementptr i8, ptr %A, i64 %iv + call void @use(ptr %gep) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, %wide.trip.count + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @udiv4_and_udiv2_mul_4_dividend_multiple_of_2(i1 %c, ptr %A) { +; CHECK-LABEL: 'udiv4_and_udiv2_mul_4_dividend_multiple_of_2' +; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2_mul_4_dividend_multiple_of_2 +; CHECK-NEXT: %start = select i1 %c, i32 514, i32 0 +; CHECK-NEXT: --> %start U: [0,515) S: [0,515) +; CHECK-NEXT: %div.2 = lshr i32 %start, 1 +; CHECK-NEXT: --> (%start /u 2) U: [0,258) S: [0,258) +; CHECK-NEXT: %div.4 = lshr i32 %start, 2 +; CHECK-NEXT: --> (%start /u 4) U: [0,129) S: [0,129) +; CHECK-NEXT: %iv.start = zext i32 %div.4 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 4) U: [0,129) S: [0,129) +; CHECK-NEXT: %wide.trip.count = zext i32 %div.2 to i64 +; CHECK-NEXT: --> ((zext i32 %start to i64) /u 2) U: [0,258) S: [0,258) +; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {((zext i32 %start to i64) /u 4),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 2) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep = getelementptr i32, ptr %A, i64 %iv +; CHECK-NEXT: --> {((4 * ((zext i32 %start to i64) /u 4)) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((zext i32 %start to i64) /u 2)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = add i64 %iv, 1 +; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 4)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 2)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2_mul_4_dividend_multiple_of_2 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %start = select i1 %c, i32 514, i32 0 + %div.2 = lshr i32 %start, 1 + %div.4 = lshr i32 %start, 2 + %iv.start = zext i32 %div.4 to i64 + %wide.trip.count = zext i32 %div.2 to i64 + br label %loop + +loop: + %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] + %gep = getelementptr i32, ptr %A, i64 %iv + call void @use(ptr %gep) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, %wide.trip.count + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + + From 49185126023beed4398f953bdf86bfa336628e77 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Sep 2025 16:33:40 +0100 Subject: [PATCH 27/60] [SCEV] Cover more multipler/divisor combinations in folding test. Consolidate tests for multiple divisors in a single loop, add multiplies by 1, 2, 5, 6. Extends test coverage for https://github.com/llvm/llvm-project/pull/157159. (cherry picked from commit b9f571f834742e05d4edca67e4733bc1d40ec58c) --- .../ScalarEvolution/mul-udiv-folds.ll | 146 +++++------------- 1 file changed, 42 insertions(+), 104 deletions(-) diff --git a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll index 9f4360d2ae383..b5d976ba3acda 100644 --- a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll @@ -3,9 +3,9 @@ declare void @use(ptr) -define void @udiv4_and_udiv2_mul_4(i1 %c, ptr %A) { -; CHECK-LABEL: 'udiv4_and_udiv2_mul_4' -; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2_mul_4 +define void @udiv4_and_udiv2(i1 %c, ptr %A) { +; CHECK-LABEL: 'udiv4_and_udiv2' +; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2 ; CHECK-NEXT: %start = select i1 %c, i32 512, i32 0 ; CHECK-NEXT: --> %start U: [0,513) S: [0,513) ; CHECK-NEXT: %div.2 = lshr i32 %start, 1 @@ -18,56 +18,19 @@ define void @udiv4_and_udiv2_mul_4(i1 %c, ptr %A) { ; CHECK-NEXT: --> ((zext i32 %start to i64) /u 2) U: [0,257) S: [0,257) ; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] ; CHECK-NEXT: --> {((zext i32 %start to i64) /u 4),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 2) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: %gep = getelementptr i32, ptr %A, i64 %iv -; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + (4 * ((zext i32 %start to i64) /u 2)) + (-4 * ((zext i32 %start to i64) /u 4)) + %A) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: %iv.next = add i64 %iv, 1 -; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 4)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 2)) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2_mul_4 -; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) -; CHECK-NEXT: Loop %loop: Trip multiple is 1 -; -entry: - %start = select i1 %c, i32 512, i32 0 - %div.2 = lshr i32 %start, 1 - %div.4 = lshr i32 %start, 2 - %iv.start = zext i32 %div.4 to i64 - %wide.trip.count = zext i32 %div.2 to i64 - br label %loop - -loop: - %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] - %gep = getelementptr i32, ptr %A, i64 %iv - call void @use(ptr %gep) - %iv.next = add i64 %iv, 1 - %ec = icmp eq i64 %iv, %wide.trip.count - br i1 %ec, label %exit, label %loop - -exit: - ret void -} - -define void @udiv4_and_udiv2_mul_1(i1 %c, ptr %A) { -; CHECK-LABEL: 'udiv4_and_udiv2_mul_1' -; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2_mul_1 -; CHECK-NEXT: %start = select i1 %c, i32 512, i32 0 -; CHECK-NEXT: --> %start U: [0,513) S: [0,513) -; CHECK-NEXT: %div.2 = lshr i32 %start, 1 -; CHECK-NEXT: --> (%start /u 2) U: [0,257) S: [0,257) -; CHECK-NEXT: %div.4 = lshr i32 %start, 2 -; CHECK-NEXT: --> (%start /u 4) U: [0,129) S: [0,129) -; CHECK-NEXT: %iv.start = zext i32 %div.4 to i64 -; CHECK-NEXT: --> ((zext i32 %start to i64) /u 4) U: [0,129) S: [0,129) -; CHECK-NEXT: %wide.trip.count = zext i32 %div.2 to i64 -; CHECK-NEXT: --> ((zext i32 %start to i64) /u 2) U: [0,257) S: [0,257) -; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {((zext i32 %start to i64) /u 4),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 2) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: %gep = getelementptr i8, ptr %A, i64 %iv +; CHECK-NEXT: %gep.8 = getelementptr i8, ptr %A, i64 %iv ; CHECK-NEXT: --> {(((zext i32 %start to i64) /u 4) + %A),+,1}<%loop> U: full-set S: full-set Exits: (((zext i32 %start to i64) /u 2) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.16 = getelementptr i16, ptr %A, i64 %iv +; CHECK-NEXT: --> {((2 * ((zext i32 %start to i64) /u 4)) + %A),+,2}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.32 = getelementptr i32, ptr %A, i64 %iv +; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + (4 * ((zext i32 %start to i64) /u 2)) + (-4 * ((zext i32 %start to i64) /u 4)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.40 = getelementptr <{ i32, i8 }>, ptr %A, i64 %iv +; CHECK-NEXT: --> {((5 * ((zext i32 %start to i64) /u 4)) + %A),+,5}<%loop> U: full-set S: full-set Exits: ((5 * ((zext i32 %start to i64) /u 2)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.48 = getelementptr <{ i32, i16 }>, ptr %A, i64 %iv +; CHECK-NEXT: --> {((6 * ((zext i32 %start to i64) /u 4)) + %A),+,6}<%loop> U: full-set S: full-set Exits: ((6 * ((zext i32 %start to i64) /u 2)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = add i64 %iv, 1 ; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 4)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 2)) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2_mul_1 +; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2 ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) @@ -83,8 +46,16 @@ entry: loop: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] - %gep = getelementptr i8, ptr %A, i64 %iv - call void @use(ptr %gep) + %gep.8 = getelementptr i8, ptr %A, i64 %iv + call void @use(ptr %gep.8) + %gep.16 = getelementptr i16, ptr %A, i64 %iv + call void @use(ptr %gep.16) + %gep.32 = getelementptr i32, ptr %A, i64 %iv + call void @use(ptr %gep.32) + %gep.40 = getelementptr <{i32, i8}>, ptr %A, i64 %iv + call void @use(ptr %gep.40) + %gep.48 = getelementptr <{i32, i16}>, ptr %A, i64 %iv + call void @use(ptr %gep.48) %iv.next = add i64 %iv, 1 %ec = icmp eq i64 %iv, %wide.trip.count br i1 %ec, label %exit, label %loop @@ -92,8 +63,6 @@ loop: exit: ret void } - - define void @udiv3_and_udiv5_mul_4(i1 %c, ptr %A) { ; CHECK-LABEL: 'udiv3_and_udiv5_mul_4' ; CHECK-NEXT: Classifying expressions for: @udiv3_and_udiv5_mul_4 @@ -109,8 +78,16 @@ define void @udiv3_and_udiv5_mul_4(i1 %c, ptr %A) { ; CHECK-NEXT: --> ((zext i32 %start to i64) /u 3) U: [0,171) S: [0,171) ; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] ; CHECK-NEXT: --> {((zext i32 %start to i64) /u 5),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 3) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: %gep = getelementptr i8, ptr %A, i64 %iv +; CHECK-NEXT: %gep.8 = getelementptr i8, ptr %A, i64 %iv ; CHECK-NEXT: --> {(((zext i32 %start to i64) /u 5) + %A),+,1}<%loop> U: full-set S: full-set Exits: (((zext i32 %start to i64) /u 3) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.16 = getelementptr i16, ptr %A, i64 %iv +; CHECK-NEXT: --> {((2 * ((zext i32 %start to i64) /u 5)) + %A),+,2}<%loop> U: full-set S: full-set Exits: ((2 * ((zext i32 %start to i64) /u 3)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.32 = getelementptr i32, ptr %A, i64 %iv +; CHECK-NEXT: --> {((4 * ((zext i32 %start to i64) /u 5)) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((zext i32 %start to i64) /u 3)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.40 = getelementptr <{ i32, i8 }>, ptr %A, i64 %iv +; CHECK-NEXT: --> {((5 * ((zext i32 %start to i64) /u 5)) + %A),+,5}<%loop> U: full-set S: full-set Exits: ((5 * ((zext i32 %start to i64) /u 3)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.48 = getelementptr <{ i32, i16 }>, ptr %A, i64 %iv +; CHECK-NEXT: --> {((6 * ((zext i32 %start to i64) /u 5)) + %A),+,6}<%loop> U: full-set S: full-set Exits: ((6 * ((zext i32 %start to i64) /u 3)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = add i64 %iv, 1 ; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 5)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 3)) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @udiv3_and_udiv5_mul_4 @@ -129,8 +106,16 @@ entry: loop: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] - %gep = getelementptr i8, ptr %A, i64 %iv - call void @use(ptr %gep) + %gep.8 = getelementptr i8, ptr %A, i64 %iv + call void @use(ptr %gep.8) + %gep.16 = getelementptr i16, ptr %A, i64 %iv + call void @use(ptr %gep.16) + %gep.32 = getelementptr i32, ptr %A, i64 %iv + call void @use(ptr %gep.32) + %gep.40 = getelementptr <{i32, i8}>, ptr %A, i64 %iv + call void @use(ptr %gep.40) + %gep.48 = getelementptr <{i32, i16}>, ptr %A, i64 %iv + call void @use(ptr %gep.48) %iv.next = add i64 %iv, 1 %ec = icmp eq i64 %iv, %wide.trip.count br i1 %ec, label %exit, label %loop @@ -138,50 +123,3 @@ loop: exit: ret void } - -define void @udiv4_and_udiv2_mul_4_dividend_multiple_of_2(i1 %c, ptr %A) { -; CHECK-LABEL: 'udiv4_and_udiv2_mul_4_dividend_multiple_of_2' -; CHECK-NEXT: Classifying expressions for: @udiv4_and_udiv2_mul_4_dividend_multiple_of_2 -; CHECK-NEXT: %start = select i1 %c, i32 514, i32 0 -; CHECK-NEXT: --> %start U: [0,515) S: [0,515) -; CHECK-NEXT: %div.2 = lshr i32 %start, 1 -; CHECK-NEXT: --> (%start /u 2) U: [0,258) S: [0,258) -; CHECK-NEXT: %div.4 = lshr i32 %start, 2 -; CHECK-NEXT: --> (%start /u 4) U: [0,129) S: [0,129) -; CHECK-NEXT: %iv.start = zext i32 %div.4 to i64 -; CHECK-NEXT: --> ((zext i32 %start to i64) /u 4) U: [0,129) S: [0,129) -; CHECK-NEXT: %wide.trip.count = zext i32 %div.2 to i64 -; CHECK-NEXT: --> ((zext i32 %start to i64) /u 2) U: [0,258) S: [0,258) -; CHECK-NEXT: %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {((zext i32 %start to i64) /u 4),+,1}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) /u 2) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: %gep = getelementptr i32, ptr %A, i64 %iv -; CHECK-NEXT: --> {((4 * ((zext i32 %start to i64) /u 4)) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((zext i32 %start to i64) /u 2)) + %A) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: %iv.next = add i64 %iv, 1 -; CHECK-NEXT: --> {(1 + ((zext i32 %start to i64) /u 4)),+,1}<%loop> U: full-set S: full-set Exits: (1 + ((zext i32 %start to i64) /u 2)) LoopDispositions: { %loop: Computable } -; CHECK-NEXT: Determining loop execution counts for: @udiv4_and_udiv2_mul_4_dividend_multiple_of_2 -; CHECK-NEXT: Loop %loop: backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -1 -; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-1 * ((zext i32 %start to i64) /u 4)) + ((zext i32 %start to i64) /u 2)) -; CHECK-NEXT: Loop %loop: Trip multiple is 1 -; -entry: - %start = select i1 %c, i32 514, i32 0 - %div.2 = lshr i32 %start, 1 - %div.4 = lshr i32 %start, 2 - %iv.start = zext i32 %div.4 to i64 - %wide.trip.count = zext i32 %div.2 to i64 - br label %loop - -loop: - %iv = phi i64 [ %iv.start, %entry ], [ %iv.next, %loop ] - %gep = getelementptr i32, ptr %A, i64 %iv - call void @use(ptr %gep) - %iv.next = add i64 %iv, 1 - %ec = icmp eq i64 %iv, %wide.trip.count - br i1 %ec, label %exit, label %loop - -exit: - ret void -} - - From c38a2e922fafce618f5dbcd3fe984596ba1ec8bd Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 9 Sep 2025 10:41:34 +0100 Subject: [PATCH 28/60] [SCEV] Generalize (C * A /u C) -> A fold to (C1 * A /u C2) -> C1/C2 * A. (#157159) Generalize fold added in 74ec38fad0a1289 (https://github.com/llvm/llvm-project/pull/156730) to support multiplying and dividing by different constants, given they are both powers-of-2 and C1 is a multiple of C2, checked via logBase2. https://alive2.llvm.org/ce/z/eqJ2xj PR: https://github.com/llvm/llvm-project/pull/157159 (cherry picked from commit a1afe66f328d4f77c7b5a9ad7b55b8349b1c7017) --- llvm/lib/Analysis/ScalarEvolution.cpp | 15 +++++++++------ .../Analysis/ScalarEvolution/mul-udiv-folds.ll | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 856c13ec9e9e1..e8d8ce76b1382 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3216,13 +3216,16 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, }; } - // Try to fold (C * D /u C) -> D, if C is a power-of-2 and D is a multiple - // of C. + // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2, + // D is a multiple of C2, and C1 is a multiple of C1. const SCEV *D; - if (match(Ops[1], m_scev_UDiv(m_SCEV(D), m_scev_Specific(LHSC))) && - LHSC->getAPInt().isPowerOf2() && - LHSC->getAPInt().logBase2() <= getMinTrailingZeros(D)) { - return D; + const SCEVConstant *C2; + const APInt &LHSV = LHSC->getAPInt(); + if (LHSV.isPowerOf2() && + match(Ops[1], m_scev_UDiv(m_SCEV(D), m_SCEVConstant(C2))) && + C2->getAPInt().isPowerOf2() && LHSV.uge(C2->getAPInt()) && + LHSV.logBase2() <= getMinTrailingZeros(D)) { + return getMulExpr(getUDivExpr(LHSC, C2), D); } } } diff --git a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll index b5d976ba3acda..dfaf0c95bc2f8 100644 --- a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll @@ -23,7 +23,7 @@ define void @udiv4_and_udiv2(i1 %c, ptr %A) { ; CHECK-NEXT: %gep.16 = getelementptr i16, ptr %A, i64 %iv ; CHECK-NEXT: --> {((2 * ((zext i32 %start to i64) /u 4)) + %A),+,2}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.32 = getelementptr i32, ptr %A, i64 %iv -; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + (4 * ((zext i32 %start to i64) /u 2)) + (-4 * ((zext i32 %start to i64) /u 4)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((3 * (zext i32 %start to i64)) + (-4 * ((zext i32 %start to i64) /u 4)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.40 = getelementptr <{ i32, i8 }>, ptr %A, i64 %iv ; CHECK-NEXT: --> {((5 * ((zext i32 %start to i64) /u 4)) + %A),+,5}<%loop> U: full-set S: full-set Exits: ((5 * ((zext i32 %start to i64) /u 2)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.48 = getelementptr <{ i32, i16 }>, ptr %A, i64 %iv From af12a1311cf142bcdc4c535d3b73d79833be371c Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 10 Sep 2025 14:59:14 +0100 Subject: [PATCH 29/60] [SCEV] Fold ((-1 * C1) * D / C1) -> -1 * D. (#157555) Treat negative constants C as -1 * abs(C1) when folding multiplies and udivs. Alive2 Proof: https://alive2.llvm.org/ce/z/bdj9W2 PR: https://github.com/llvm/llvm-project/pull/157555 (cherry picked from commit 6580c9102f26789c655d6945372dd259b9e52e33) --- llvm/lib/Analysis/ScalarEvolution.cpp | 16 ++++++++++------ .../Analysis/ScalarEvolution/mul-udiv-folds.ll | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index e8d8ce76b1382..a22ab35d8acc2 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3217,15 +3217,19 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, } // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2, - // D is a multiple of C2, and C1 is a multiple of C1. + // D is a multiple of C2, and C1 is a multiple of C2. const SCEV *D; + APInt C1V = LHSC->getAPInt(); + // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. + if (C1V.isNegative() && !C1V.isMinSignedValue()) + C1V = C1V.abs(); const SCEVConstant *C2; - const APInt &LHSV = LHSC->getAPInt(); - if (LHSV.isPowerOf2() && + if (C1V.isPowerOf2() && match(Ops[1], m_scev_UDiv(m_SCEV(D), m_SCEVConstant(C2))) && - C2->getAPInt().isPowerOf2() && LHSV.uge(C2->getAPInt()) && - LHSV.logBase2() <= getMinTrailingZeros(D)) { - return getMulExpr(getUDivExpr(LHSC, C2), D); + C2->getAPInt().isPowerOf2() && C1V.uge(C2->getAPInt()) && + C1V.logBase2() <= getMinTrailingZeros(D)) { + const SCEV *NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D); + return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul); } } } diff --git a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll index dfaf0c95bc2f8..8dd8ec47e7090 100644 --- a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll @@ -23,7 +23,7 @@ define void @udiv4_and_udiv2(i1 %c, ptr %A) { ; CHECK-NEXT: %gep.16 = getelementptr i16, ptr %A, i64 %iv ; CHECK-NEXT: --> {((2 * ((zext i32 %start to i64) /u 4)) + %A),+,2}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.32 = getelementptr i32, ptr %A, i64 %iv -; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((3 * (zext i32 %start to i64)) + (-4 * ((zext i32 %start to i64) /u 4)) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((2 * (zext i32 %start to i64)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.40 = getelementptr <{ i32, i8 }>, ptr %A, i64 %iv ; CHECK-NEXT: --> {((5 * ((zext i32 %start to i64) /u 4)) + %A),+,5}<%loop> U: full-set S: full-set Exits: ((5 * ((zext i32 %start to i64) /u 2)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.48 = getelementptr <{ i32, i16 }>, ptr %A, i64 %iv From ca229b82728d257d0ee146dc142f8eba4e9fa92b Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 11 Sep 2025 09:08:47 +0100 Subject: [PATCH 30/60] [SCEV] Fold (C1 * A /u C2) -> A /u (C2 /u C1), if C2 > C1. (#157656) If C2 >u C1 and C1 >u 1, fold to A /u (C2 /u C1). Depends on https://github.com/llvm/llvm-project/pull/157555. Alive2 Proof: https://alive2.llvm.org/ce/z/BWvQYN PR: https://github.com/llvm/llvm-project/pull/157656 (cherry picked from commit 70012fda6312ba87bc0bf9009402e0869a816d1f) --- llvm/lib/Analysis/ScalarEvolution.cpp | 18 +++++++++++++----- .../Analysis/ScalarEvolution/mul-udiv-folds.ll | 2 +- .../LoopStrengthReduce/duplicated-phis.ll | 3 +-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index a22ab35d8acc2..bfbfbd02d4d8c 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3217,18 +3217,26 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, } // Try to fold (C1 * D /u C2) -> C1/C2 * D, if C1 and C2 are powers-of-2, - // D is a multiple of C2, and C1 is a multiple of C2. + // D is a multiple of C2, and C1 is a multiple of C2. If C2 is a multiple + // of C1, fold to (D /u (C2 /u C1)). const SCEV *D; APInt C1V = LHSC->getAPInt(); - // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. - if (C1V.isNegative() && !C1V.isMinSignedValue()) + // (C1 * D /u C2) == -1 * -C1 * D /u C2 when C1 != INT_MIN. Don't treat -1 + // as -1 * 1, as it won't enable additional folds. + if (C1V.isNegative() && !C1V.isMinSignedValue() && !C1V.isAllOnes()) C1V = C1V.abs(); const SCEVConstant *C2; if (C1V.isPowerOf2() && match(Ops[1], m_scev_UDiv(m_SCEV(D), m_SCEVConstant(C2))) && - C2->getAPInt().isPowerOf2() && C1V.uge(C2->getAPInt()) && + C2->getAPInt().isPowerOf2() && C1V.logBase2() <= getMinTrailingZeros(D)) { - const SCEV *NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D); + const SCEV *NewMul; + if (C1V.uge(C2->getAPInt())) { + NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D); + } else { + assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier"); + NewMul = getUDivExpr(D, getUDivExpr(C2, getConstant(C1V))); + } return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul); } } diff --git a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll index 8dd8ec47e7090..1d34706baadeb 100644 --- a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll @@ -21,7 +21,7 @@ define void @udiv4_and_udiv2(i1 %c, ptr %A) { ; CHECK-NEXT: %gep.8 = getelementptr i8, ptr %A, i64 %iv ; CHECK-NEXT: --> {(((zext i32 %start to i64) /u 4) + %A),+,1}<%loop> U: full-set S: full-set Exits: (((zext i32 %start to i64) /u 2) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.16 = getelementptr i16, ptr %A, i64 %iv -; CHECK-NEXT: --> {((2 * ((zext i32 %start to i64) /u 4)) + %A),+,2}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + %A) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(((zext i32 %start to i64) /u 2) + %A),+,2}<%loop> U: full-set S: full-set Exits: ((zext i32 %start to i64) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.32 = getelementptr i32, ptr %A, i64 %iv ; CHECK-NEXT: --> {((zext i32 %start to i64) + %A),+,4}<%loop> U: full-set S: full-set Exits: ((2 * (zext i32 %start to i64)) + %A) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %gep.40 = getelementptr <{ i32, i8 }>, ptr %A, i64 %iv diff --git a/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll b/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll index cee8c8abdb450..c59f7d9c2a41a 100644 --- a/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll +++ b/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll @@ -18,8 +18,7 @@ define i64 @test_duplicated_phis(i64 noundef %N) { ; CHECK: [[FOR_BODY_PREHEADER_NEW]]: ; CHECK-NEXT: [[UNROLL_ITER:%.*]] = and i64 [[MUL]], -4 ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[UNROLL_ITER]], -4 -; CHECK-NEXT: [[TMP5:%.*]] = lshr i64 [[TMP4]], 2 -; CHECK-NEXT: [[TMP3:%.*]] = shl nuw nsw i64 [[TMP5]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = lshr i64 [[TMP4]], 1 ; CHECK-NEXT: [[LSR_IV_NEXT:%.*]] = sub i64 -3, [[TMP3]] ; CHECK-NEXT: br label %[[FOR_BODY:.*]] ; CHECK: [[FOR_BODY]]: From 41b6b30a3335291419a6eaafd5a945b352865725 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Sep 2025 12:56:25 +0100 Subject: [PATCH 31/60] [SCEV] Add more tests for MUL/UDIV folds. Add additional test coverage for https://github.com/llvm/llvm-project/pull/157656 covering cases where the dividend is not a multiple of the divisor, causing the revert of 70012fda631. (cherry picked from commit 6a726e9a4d3d05f9aecf366fb6488d63135f04f3) --- .../ScalarEvolution/mul-udiv-folds.ll | 65 +++++++++++++++++++ .../LoopStrengthReduce/duplicated-phis.ll | 38 +++++++++++ 2 files changed, 103 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll index 1d34706baadeb..1e21fbf08a92f 100644 --- a/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll +++ b/llvm/test/Analysis/ScalarEvolution/mul-udiv-folds.ll @@ -123,3 +123,68 @@ loop: exit: ret void } + +declare void @use.i64(i64) + +define void @dividend_not_known_multiple_of_divisor(i64 %x) { +; CHECK-LABEL: 'dividend_not_known_multiple_of_divisor' +; CHECK-NEXT: Classifying expressions for: @dividend_not_known_multiple_of_divisor +; CHECK-NEXT: %mul.2 = shl i64 %x, 1 +; CHECK-NEXT: --> (2 * %x) U: [0,-1) S: [-9223372036854775808,9223372036854775807) +; CHECK-NEXT: %div.16 = lshr exact i64 %mul.2, 4 +; CHECK-NEXT: --> ((2 * %x) /u 16) U: [0,1152921504606846976) S: [0,1152921504606846976) +; CHECK-NEXT: %m2 = and i64 %div.16, 1152921504606846974 +; CHECK-NEXT: --> (2 * ((2 * %x) /u 32)) U: [0,1152921504606846975) S: [0,1152921504606846975) +; CHECK-NEXT: %m3 = mul i64 %div.16, 2 +; CHECK-NEXT: --> (2 * ((2 * %x) /u 16)) U: [0,2305843009213693951) S: [0,2305843009213693951) +; CHECK-NEXT: %m4 = udiv i64 %m3, 4 +; CHECK-NEXT: --> ((2 * ((2 * %x) /u 16)) /u 4) U: [0,576460752303423488) S: [0,576460752303423488) +; CHECK-NEXT: Determining loop execution counts for: @dividend_not_known_multiple_of_divisor +; +entry: + %mul.2 = shl i64 %x, 1 + %div.16 = lshr exact i64 %mul.2, 4 + %m2 = and i64 %div.16, 1152921504606846974 + call void @use.i64(i64 %m2) + + %m3 = mul i64 %div.16, 2 + %m4 = udiv i64 %m3, 4 + call void @use.i64(i64 %m4) + ret void +} + +define void @btc_depends_on_div_mul(i64 %x) { +; CHECK-LABEL: 'btc_depends_on_div_mul' +; CHECK-NEXT: Classifying expressions for: @btc_depends_on_div_mul +; CHECK-NEXT: %mul.2 = shl i64 %x, 1 +; CHECK-NEXT: --> (2 * %x) U: [0,-1) S: [-9223372036854775808,9223372036854775807) +; CHECK-NEXT: %div.16 = lshr exact i64 %mul.2, 4 +; CHECK-NEXT: --> ((2 * %x) /u 16) U: [0,1152921504606846976) S: [0,1152921504606846976) +; CHECK-NEXT: %masked = and i64 %div.16, 1152921504606846974 +; CHECK-NEXT: --> (2 * ((2 * %x) /u 32)) U: [0,1152921504606846975) S: [0,1152921504606846975) +; CHECK-NEXT: %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {0,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: (-2 + (2 * ((2 * %x) /u 32))) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = add i64 %iv, 2 +; CHECK-NEXT: --> {2,+,2}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: (2 * ((2 * %x) /u 32)) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @btc_depends_on_div_mul +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-2 + (2 * ((2 * %x) /u 32))) /u 2) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 9223372036854775807 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-2 + (2 * ((2 * %x) /u 32))) /u 2) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %mul.2 = shl i64 %x, 1 + %div.16 = lshr exact i64 %mul.2, 4 + %masked = and i64 %div.16, 1152921504606846974 + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop ] + call void @use.i64(i64 %iv) + %iv.next = add i64 %iv, 2 + %ec = icmp eq i64 %iv.next, %masked + br i1 %ec, label %exit, label %loop + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll b/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll index c59f7d9c2a41a..43389b5df8f00 100644 --- a/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll +++ b/llvm/test/Transforms/LoopStrengthReduce/duplicated-phis.ll @@ -83,3 +83,41 @@ for.end: %res.0.lcssa = phi i64 [ 0, %entry ], [ %spec.select, %for.end.loopexit.unr-lcssa ] ret i64 %res.0.lcssa } + +define i64 @duplicated_phis_compare_uses_mul_udiv(i64 %x) { +; CHECK-LABEL: define i64 @duplicated_phis_compare_uses_mul_udiv( +; CHECK-SAME: i64 [[X:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[MUL_2:%.*]] = shl i64 [[X]], 1 +; CHECK-NEXT: [[DIV_16:%.*]] = lshr i64 [[MUL_2]], 4 +; CHECK-NEXT: [[MASKED:%.*]] = and i64 [[DIV_16]], 1152921504606846974 +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV_1:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_1_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: call void @clobber() +; CHECK-NEXT: [[IV_1_NEXT]] = add i64 [[IV_1]], 2 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[MASKED]], [[IV_1_NEXT]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i64 [[IV_1_NEXT]] +; +entry: + %mul.2 = shl i64 %x, 1 + %div.16 = lshr exact i64 %mul.2, 4 + %masked = and i64 %div.16, 1152921504606846974 + br label %loop + +loop: + %iv.1 = phi i64 [ 0, %entry ], [ %iv.1.next, %loop ] + %iv.2 = phi i64 [ 0, %entry ], [ %iv.2.next, %loop ] + call void @clobber() + %iv.1.next = add i64 %iv.1, 2 + %iv.2.next = add i64 %iv.2, 2 + %ec = icmp eq i64 %iv.2.next, %masked + br i1 %ec, label %exit, label %loop + +exit: + ret i64 %iv.1.next +} + +declare void @clobber() From d3587ecaf8f129aea32442147207fcdf92d98cdb Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 17 Sep 2025 14:28:29 +0100 Subject: [PATCH 32/60] Reapply "[SCEV] Fold (C1 * A /u C2) -> A /u (C2 /u C1), if C2 > C1." (#158328) This reverts commit fd58f235f8c5bd40d98acfd8e7fb11d41de301c7. The recommit contains an extra check to make sure that D is a multiple of C2, if C2 > C1. This fixes the issue causing the revert fd58f235f8c. Tests have been added in 6a726e9a4d3d0. Original message: If C2 >u C1 and C1 >u 1, fold to A /u (C2 /u C1). Depends on https://github.com/llvm/llvm-project/pull/157555. Alive2 Proof: https://alive2.llvm.org/ce/z/BWvQYN PR: https://github.com/llvm/llvm-project/pull/157656 (cherry picked from commit f78150d2d477b31b46d1afdd255020689f2ddccf) --- llvm/lib/Analysis/ScalarEvolution.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index bfbfbd02d4d8c..0b13dfb1617f1 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -3230,14 +3230,15 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, match(Ops[1], m_scev_UDiv(m_SCEV(D), m_SCEVConstant(C2))) && C2->getAPInt().isPowerOf2() && C1V.logBase2() <= getMinTrailingZeros(D)) { - const SCEV *NewMul; + const SCEV *NewMul = nullptr; if (C1V.uge(C2->getAPInt())) { NewMul = getMulExpr(getUDivExpr(getConstant(C1V), C2), D); - } else { + } else if (C2->getAPInt().logBase2() <= getMinTrailingZeros(D)) { assert(C1V.ugt(1) && "C1 <= 1 should have been folded earlier"); NewMul = getUDivExpr(D, getUDivExpr(C2, getConstant(C1V))); } - return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul); + if (NewMul) + return C1V == LHSC->getAPInt() ? NewMul : getNegativeSCEV(NewMul); } } } From ad95f8503564e722cbc5e8eff16ccd324f7f72bc Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 21 Aug 2025 16:36:25 +0100 Subject: [PATCH 33/60] [InstComb] Fold inttoptr (add (ptrtoint %B), %O) -> GEP for ICMP users. (#153421) Replace inttoptr (add (ptrtoint %B), %O) with (getelementptr i8, %B, %o) if all users are ICmp instruction, which in turn means only the address value is compared. We should be able to do this, if the src pointer, the integer type and the destination pointer types have the same bitwidth and address space. A common source of such (inttoptr (add (ptrtoint %B), %O)) is from various iterations in libc++. In practice this triggers in a number of files in Clang and various open source projects, including cppcheck, diamond, llama and more. Alive2 Proof with constant offset: https://alive2.llvm.org/ce/z/K_5N_B PR: https://github.com/llvm/llvm-project/pull/153421 (cherry picked from commit 1b0b59ae4343500d52593c7bd3bce550b3880f7d) --- .../InstCombine/InstCombineCasts.cpp | 13 ++ .../InstCombine/fold-bin-operand.ll | 220 ++++++++++++++++++ 2 files changed, 233 insertions(+) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index 033ef8be700eb..e0d6e77eb53c8 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -2069,6 +2069,19 @@ Instruction *InstCombinerImpl::visitIntToPtr(IntToPtrInst &CI) { return new IntToPtrInst(P, CI.getType()); } + // Replace (inttoptr (add (ptrtoint %Base), %Offset)) with + // (getelementptr i8, %Base, %Offset) if all users are ICmps. + Value *Base; + Value *Offset; + if (match(CI.getOperand(0), + m_OneUse(m_c_Add(m_PtrToIntSameSize(DL, m_Value(Base)), + m_Value(Offset)))) && + CI.getType()->getPointerAddressSpace() == + Base->getType()->getPointerAddressSpace() && + all_of(CI.users(), IsaPred)) { + return GetElementPtrInst::Create(Builder.getInt8Ty(), Base, Offset); + } + if (Instruction *I = commonCastTransforms(CI)) return I; diff --git a/llvm/test/Transforms/InstCombine/fold-bin-operand.ll b/llvm/test/Transforms/InstCombine/fold-bin-operand.ll index f28262b2a77e0..0023a10312451 100644 --- a/llvm/test/Transforms/InstCombine/fold-bin-operand.ll +++ b/llvm/test/Transforms/InstCombine/fold-bin-operand.ll @@ -30,6 +30,226 @@ define i32 @g(i32 %x) { ret i32 %b } +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp(ptr %src, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp( +; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10 +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %c = icmp eq ptr %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_operands_swapped(ptr %src, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_operands_swapped( +; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10 +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 10, %i + %p = inttoptr i64 %a to ptr + %c = icmp eq ptr %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset(ptr %src, i64 %off, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset( +; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFF:%.*]] +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, %off + %p = inttoptr i64 %a to ptr + %c = icmp eq ptr %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset_operands_swapped(ptr %src, i64 %off, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_constant_offset_operands_swapped( +; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[OFF:%.*]] +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %off, %i + %p = inttoptr i64 %a to ptr + %c = icmp eq ptr %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_different_src_address_spaces(ptr addrspace(1) %src, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_different_src_address_spaces( +; CHECK-NEXT: [[I:%.*]] = ptrtoint ptr addrspace(1) [[SRC:%.*]] to i64 +; CHECK-NEXT: [[A:%.*]] = add i64 [[I]], 10 +; CHECK-NEXT: [[P:%.*]] = inttoptr i64 [[A]] to ptr +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2:%.*]], [[P]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr addrspace(1) %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %c = icmp eq ptr %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_different_dst_address_spaces(ptr %src, ptr addrspace(1) %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_different_dst_address_spaces( +; CHECK-NEXT: [[I:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64 +; CHECK-NEXT: [[A:%.*]] = add i64 [[I]], 10 +; CHECK-NEXT: [[P:%.*]] = inttoptr i64 [[A]] to ptr addrspace(1) +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr addrspace(1) [[P2:%.*]], [[P]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr addrspace(1) + %c = icmp eq ptr addrspace(1) %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_int_type_does_not_match_ptr_ty(ptr %src, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_int_type_does_not_match_ptr_ty( +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64 +; CHECK-NEXT: [[I:%.*]] = trunc i64 [[TMP1]] to i8 +; CHECK-NEXT: [[A:%.*]] = add i8 [[I]], 10 +; CHECK-NEXT: [[TMP2:%.*]] = zext i8 [[A]] to i64 +; CHECK-NEXT: [[P:%.*]] = inttoptr i64 [[TMP2]] to ptr +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2:%.*]], [[P]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i8 + %a = add i8 %i, 10 + %p = inttoptr i8 %a to ptr + %c = icmp eq ptr %p, %p2 + ret i1 %c +} + +define i1 @inttoptr_add_multiple_users_ptrtoint_used_by_single_icmp(ptr %src, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_multiple_users_ptrtoint_used_by_single_icmp( +; CHECK-NEXT: [[I:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64 +; CHECK-NEXT: [[A:%.*]] = add i64 [[I]], 10 +; CHECK-NEXT: [[P:%.*]] = inttoptr i64 [[A]] to ptr +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2:%.*]], [[P]] +; CHECK-NEXT: call void @bar(i64 [[A]]) +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %c = icmp eq ptr %p, %p2 + call void @bar(i64 %a) + ret i1 %c +} + +define i1 @multiple_inttoptr_add_ptrtoint_used_by_single_icmp(ptr %src) { +; CHECK-LABEL: @multiple_inttoptr_add_ptrtoint_used_by_single_icmp( +; CHECK-NEXT: ret i1 false +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %a.2 = add i64 %i, 11 + %p.2 = inttoptr i64 %a.2 to ptr + %c = icmp eq ptr %p, %p.2 + ret i1 %c +} + +define i1 @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_non_constant_offset(ptr %src, i64 %off.1) { +; CHECK-LABEL: @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_non_constant_offset( +; CHECK-NEXT: ret i1 false +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %a.2 = add i64 %i, 11 + %p.2 = inttoptr i64 %a.2 to ptr + %c = icmp eq ptr %p, %p.2 + ret i1 %c +} + +define i1 @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_multiple_non_constant_offset(ptr %src, i64 %off.1, i64 %off.2) { +; CHECK-LABEL: @multiple_inttoptr_add_ptrtoint_used_by_single_icmp_multiple_non_constant_offset( +; CHECK-NEXT: [[C:%.*]] = icmp eq i64 [[OFF_1:%.*]], [[OFF_2:%.*]] +; CHECK-NEXT: ret i1 [[C]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, %off.1 + %p = inttoptr i64 %a to ptr + %a.2 = add i64 %i, %off.2 + %p.2 = inttoptr i64 %a.2 to ptr + %c = icmp eq ptr %p, %p.2 + ret i1 %c +} + +define i1 @inttoptr_add_ptrtoint_used_by_single_icmp_in_different_bb(i1 %bc, ptr %src, ptr %p2) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_single_icmp_in_different_bb( +; CHECK-NEXT: br i1 [[BC:%.*]], label [[THEN:%.*]], label [[ELSE:%.*]] +; CHECK: then: +; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10 +; CHECK-NEXT: [[C:%.*]] = icmp eq ptr [[P2]], [[P:%.*]] +; CHECK-NEXT: ret i1 [[C]] +; CHECK: else: +; CHECK-NEXT: ret i1 false +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + br i1 %bc, label %then, label %else + +then: + %c = icmp eq ptr %p, %p2 + ret i1 %c + +else: + ret i1 false +} + +define i1 @inttoptr_add_ptrtoint_used_by_multiple_icmps(ptr %src, ptr %p2, ptr %p3) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_multiple_icmps( +; CHECK-NEXT: [[P2:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 10 +; CHECK-NEXT: [[C_1:%.*]] = icmp eq ptr [[P2]], [[P:%.*]] +; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[P2]], [[P3:%.*]] +; CHECK-NEXT: [[XOR:%.*]] = xor i1 [[C_1]], [[C_2]] +; CHECK-NEXT: ret i1 [[XOR]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %c.1 = icmp eq ptr %p, %p2 + %c.2 = icmp eq ptr %p, %p3 + %xor = xor i1 %c.1, %c.2 + ret i1 %xor +} + +declare void @foo(ptr) +declare void @bar(i64) + +define i1 @inttoptr_add_ptrtoint_used_by_multiple_icmps_and_other_user(ptr %src, ptr %p2, ptr %p3) { +; CHECK-LABEL: @inttoptr_add_ptrtoint_used_by_multiple_icmps_and_other_user( +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[SRC:%.*]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add i64 [[TMP1]], 10 +; CHECK-NEXT: [[P:%.*]] = inttoptr i64 [[TMP2]] to ptr +; CHECK-NEXT: [[C_1:%.*]] = icmp eq ptr [[P2:%.*]], [[P]] +; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[P3:%.*]], [[P]] +; CHECK-NEXT: [[XOR:%.*]] = xor i1 [[C_1]], [[C_2]] +; CHECK-NEXT: call void @foo(ptr [[P]]) +; CHECK-NEXT: ret i1 [[XOR]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + %c.1 = icmp eq ptr %p, %p2 + %c.2 = icmp eq ptr %p, %p3 + %xor = xor i1 %c.1, %c.2 + call void @foo(ptr %p) + ret i1 %xor +} + define i32 @h(i1 %A, i32 %B) { ; CHECK-LABEL: @h( ; CHECK-NEXT: EntryBlock: From c0c10b1cd4d2c3dbe559ff1e80f37e0f40f1c316 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 22 Aug 2025 10:17:12 +0100 Subject: [PATCH 34/60] [InstComb] Allow more user for (add (ptrtoint %B), %O) to GEP transform. (#153566) Generalize the logic from https://github.com/llvm/llvm-project/pull/153421 to support additional cases where the pointer is only used as integer. Alive2 Proof: https://alive2.llvm.org/ce/z/po58pP This enables vectorizing std::find for some cases, if additional assumptions are provided: https://godbolt.org/z/94oq3576E Depends on https://github.com/llvm/llvm-project/pull/15342. PR: https://github.com/llvm/llvm-project/pull/153566 (cherry picked from commit 8bc038daf2d59496b33ac5825851f7c18541dab9) --- .../InstCombine/InstCombineCasts.cpp | 12 +++- .../InstCombine/inttoptr-add-phi.ll | 69 +++++++++++++++++++ .../PhaseOrdering/AArch64/std-find.ll | 4 +- 3 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp index e0d6e77eb53c8..a98ce93017a7c 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCasts.cpp @@ -2070,15 +2070,23 @@ Instruction *InstCombinerImpl::visitIntToPtr(IntToPtrInst &CI) { } // Replace (inttoptr (add (ptrtoint %Base), %Offset)) with - // (getelementptr i8, %Base, %Offset) if all users are ICmps. + // (getelementptr i8, %Base, %Offset) if the pointer is only used as integer + // value. Value *Base; Value *Offset; + auto UsesPointerAsInt = [](User *U) { + if (isa(U)) + return true; + if (auto *P = dyn_cast(U)) + return P->hasOneUse() && isa(*P->user_begin()); + return false; + }; if (match(CI.getOperand(0), m_OneUse(m_c_Add(m_PtrToIntSameSize(DL, m_Value(Base)), m_Value(Offset)))) && CI.getType()->getPointerAddressSpace() == Base->getType()->getPointerAddressSpace() && - all_of(CI.users(), IsaPred)) { + all_of(CI.users(), UsesPointerAsInt)) { return GetElementPtrInst::Create(Builder.getInt8Ty(), Base, Offset); } diff --git a/llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll b/llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll new file mode 100644 index 0000000000000..acceead23a5e5 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/inttoptr-add-phi.ll @@ -0,0 +1,69 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes=instcombine -S %s | FileCheck %s + +define i64 @inttoptr_used_by_phi_with_ptrtoint(i1 %c, ptr %src, ptr %p2) { +; CHECK-LABEL: define i64 @inttoptr_used_by_phi_with_ptrtoint( +; CHECK-SAME: i1 [[C:%.*]], ptr [[SRC:%.*]], ptr [[P2:%.*]]) { +; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: [[P:%.*]] = getelementptr i8, ptr [[SRC]], i64 10 +; CHECK-NEXT: [[TMP1:%.*]] = ptrtoint ptr [[P]] to i64 +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[ELSE]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ [[TMP1]], %[[THEN]] ], [ 0, %[[ELSE]] ] +; CHECK-NEXT: ret i64 [[PHI]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + br i1 %c, label %then, label %else + +then: + br label %exit + +else: + br label %exit + +exit: + %phi = phi ptr [ %p, %then ], [ null, %else ] + %i.2 = ptrtoint ptr %phi to i64 + ret i64 %i.2 +} + +declare void @foo(ptr) + +define i64 @inttoptr_used_by_phi_with_other_users(i1 %c, ptr %src, ptr %p2) { +; CHECK-LABEL: define i64 @inttoptr_used_by_phi_with_other_users( +; CHECK-SAME: i1 [[C:%.*]], ptr [[SRC:%.*]], ptr [[P2:%.*]]) { +; CHECK-NEXT: [[I:%.*]] = ptrtoint ptr [[SRC]] to i64 +; CHECK-NEXT: [[A:%.*]] = add i64 [[I]], 10 +; CHECK-NEXT: br i1 [[C]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[ELSE]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[PHI:%.*]] = phi i64 [ [[A]], %[[THEN]] ], [ 0, %[[ELSE]] ] +; CHECK-NEXT: [[P:%.*]] = inttoptr i64 [[A]] to ptr +; CHECK-NEXT: call void @foo(ptr [[P]]) +; CHECK-NEXT: ret i64 [[PHI]] +; + %i = ptrtoint ptr %src to i64 + %a = add i64 %i, 10 + %p = inttoptr i64 %a to ptr + br i1 %c, label %then, label %else + +then: + br label %exit + +else: + br label %exit + +exit: + %phi = phi ptr [ %p, %then ], [ null, %else ] + call void @foo(ptr %p) + %i.2 = ptrtoint ptr %phi to i64 + ret i64 %i.2 +} diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll index a3b8736a06ec7..471b1aa5dce0f 100644 --- a/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll +++ b/llvm/test/Transforms/PhaseOrdering/AArch64/std-find.ll @@ -18,8 +18,8 @@ define i64 @std_find_i16_constant_offset_with_assumptions(ptr %first.coerce, i16 ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = shl i64 [[INDEX]], 1 ; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[FIRST_COERCE]], i64 [[OFFSET_IDX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i16>, ptr [[NEXT_GEP]], align 2 -; CHECK-NEXT: [[WIDE_LOAD_FR:%.*]] = freeze <8 x i16> [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP0:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD_FR]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq <8 x i16> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP0:%.*]] = freeze <8 x i1> [[TMP9]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8 ; CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x i1> [[TMP0]] to i8 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne i8 [[TMP1]], 0 From 1be7a0584f41cd213efe06c3ed5d103636bd2700 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 3 Sep 2025 19:44:44 +0100 Subject: [PATCH 35/60] [LSR] Move test from Analysis/ScalarEvolution to Transforms, regen. Move transform test to llvm/test/Transforms/LoopStrengthReduce/X86, clean up the names a bit and regenerate check lines. (cherry picked from commit 7c10e57f54210b0ed9b863301222c58d503dedbb) --- .../ScalarEvolution/zext-signed-addrec.ll | 81 ------------------- 1 file changed, 81 deletions(-) delete mode 100644 llvm/test/Analysis/ScalarEvolution/zext-signed-addrec.ll diff --git a/llvm/test/Analysis/ScalarEvolution/zext-signed-addrec.ll b/llvm/test/Analysis/ScalarEvolution/zext-signed-addrec.ll deleted file mode 100644 index 899d31d266e51..0000000000000 --- a/llvm/test/Analysis/ScalarEvolution/zext-signed-addrec.ll +++ /dev/null @@ -1,81 +0,0 @@ -; RUN: opt -loop-reduce -S < %s | FileCheck %s -; PR18000 - -target datalayout = "e-i64:64-f80:128-s:64-n8:16:32:64-S128" -target triple = "x86_64-unknown-linux-gnu" - -@a = global i32 0, align 4 -@b = common global i32 0, align 4 -@e = common global i8 0, align 1 -@d = common global i32 0, align 4 -@c = common global i32 0, align 4 -@.str = private unnamed_addr constant [4 x i8] c"%d\0A\00", align 1 - -; Function Attrs: nounwind optsize uwtable -; CHECK-LABEL: foo -define i32 @foo() { -entry: - %.pr = load i32, ptr @b, align 4 - %cmp10 = icmp slt i32 %.pr, 1 - br i1 %cmp10, label %for.cond1.preheader.lr.ph, label %entry.for.end9_crit_edge - -entry.for.end9_crit_edge: ; preds = %entry - %.pre = load i32, ptr @c, align 4 - br label %for.end9 - -for.cond1.preheader.lr.ph: ; preds = %entry - %0 = load i32, ptr @a, align 4 - %tobool = icmp eq i32 %0, 0 - br i1 %tobool, label %for.cond1.preheader.for.cond1.preheader.split_crit_edge, label %return.loopexit.split - -for.cond1.preheader.for.cond1.preheader.split_crit_edge: ; preds = %for.cond1.preheader.lr.ph, %for.inc8 - %1 = phi i32 [ %inc, %for.inc8 ], [ %.pr, %for.cond1.preheader.lr.ph ] - br label %if.end - -; CHECK-LABEL: if.end -if.end: ; preds = %if.end, %for.cond1.preheader.for.cond1.preheader.split_crit_edge - -; CHECK: %lsr.iv = phi i32 [ %lsr.iv.next, %if.end ], [ 258, %for.cond1.preheader.for.cond1.preheader.split_crit_edge ] - %indvars.iv = phi i32 [ 1, %for.cond1.preheader.for.cond1.preheader.split_crit_edge ], [ %indvars.iv.next, %if.end ] - - %2 = phi i8 [ 1, %for.cond1.preheader.for.cond1.preheader.split_crit_edge ], [ %dec, %if.end ] - %conv7 = mul i32 %indvars.iv, 258 - %shl = and i32 %conv7, 510 - store i32 %shl, ptr @c, align 4 - -; CHECK: %lsr.iv.next = add nsw i32 %lsr.iv, -258 - %dec = add i8 %2, -1 - - %cmp2 = icmp sgt i8 %dec, -1 - %indvars.iv.next = add i32 %indvars.iv, -1 - br i1 %cmp2, label %if.end, label %for.inc8 - -for.inc8: ; preds = %if.end - store i32 0, ptr @d, align 4 - %inc = add nsw i32 %1, 1 - store i32 %inc, ptr @b, align 4 - %cmp = icmp slt i32 %1, 0 - br i1 %cmp, label %for.cond1.preheader.for.cond1.preheader.split_crit_edge, label %for.cond.for.end9_crit_edge - -for.cond.for.end9_crit_edge: ; preds = %for.inc8 - store i8 %dec, ptr @e, align 1 - br label %for.end9 - -for.end9: ; preds = %entry.for.end9_crit_edge, %for.cond.for.end9_crit_edge - %3 = phi i32 [ %.pre, %entry.for.end9_crit_edge ], [ %shl, %for.cond.for.end9_crit_edge ] - %call = tail call i32 (ptr, ...) @printf(ptr @.str, i32 %3) #2 - br label %return - -return.loopexit.split: ; preds = %for.cond1.preheader.lr.ph - store i8 1, ptr @e, align 1 - store i32 0, ptr @d, align 4 - br label %return - -return: ; preds = %return.loopexit.split, %for.end9 - %retval.0 = phi i32 [ 0, %for.end9 ], [ 1, %return.loopexit.split ] - ret i32 %retval.0 -} - -; Function Attrs: nounwind optsize -declare i32 @printf(ptr nocapture readonly, ...) - From 0d207e0ec2d4b57b94f8589a959cb60d975fa885 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 20 Sep 2025 19:24:18 +0100 Subject: [PATCH 36/60] [SCEV] Add tests that benefit from rewriting SCEVAddExpr with guards. Add additional tests benefiting from rewriting existing SCEVAddExprs with guards. (cherry picked from commit 8693ef16f6f828275c6f9b65b855504019bea27a) --- ...ge-taken-count-guard-info-apply-to-adds.ll | 29 +++++++ .../IndVarSimplify/canonicalize-cmp.ll | 84 +++++++++++++++++++ .../peel-last-iteration-with-guards.ll | 73 ++++++++++++++++ .../runtime-checks-difference.ll | 59 +++++++++++++ 4 files changed, 245 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll index 75014f3a58eb6..8df4b52757753 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll @@ -27,3 +27,32 @@ loop: exit: ret void } + +declare void @clobber() + +define void @test_add_sub_1_guard(ptr %src, i32 %n) { +; CHECK-LABEL: 'test_add_sub_1_guard' +; CHECK-NEXT: Determining loop execution counts for: @test_add_sub_1_guard +; CHECK-NEXT: Loop %loop: backedge-taken count is (zext i32 (-1 + (%n /u 2)) to i64) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4294967295 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (zext i32 (-1 + (%n /u 2)) to i64) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %shr = lshr i32 %n, 1 + %sub.1 = add i32 %shr, -1 + %sub.ext = zext i32 %sub.1 to i64 + %pre = icmp eq i32 %shr, 1 + %end = getelementptr i8, ptr %src, i64 %sub.ext + br i1 %pre, label %loop, label %exit + +loop: + %iv = phi ptr [ %src, %entry ], [ %iv.next, %loop ] + call void @clobber() + %iv.next = getelementptr i8, ptr %iv, i64 1 + %ec = icmp eq ptr %iv, %end + br i1 %ec, label %exit, label %loop + +exit: + ret void +} diff --git a/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll b/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll index 99baa6105655d..4b52479fc6c4d 100644 --- a/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll +++ b/llvm/test/Transforms/IndVarSimplify/canonicalize-cmp.ll @@ -334,4 +334,88 @@ out_of_bounds: ret i32 -1 } +define void @slt_no_smax_needed(i64 %n, ptr %dst) { +; CHECK-LABEL: @slt_no_smax_needed( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[N_TRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 +; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[N_TRUNC]], 1 +; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD_1]], 1 +; CHECK-NEXT: [[PRE:%.*]] = icmp ult i32 [[ADD_1]], 8 +; CHECK-NEXT: br i1 [[PRE]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]] +; CHECK: loop.preheader: +; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[SHR]], i32 1) +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i32 [[IV]] +; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[SMAX]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK: exit.loopexit: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %n.trunc = trunc i64 %n to i32 + %add.1 = add i32 %n.trunc, 1 + %shr = lshr i32 %add.1, 1 + %pre = icmp ult i32 %add.1, 8 + br i1 %pre, label %exit, label %loop + +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i8, ptr %dst, i32 %iv + store i8 0, ptr %gep, align 1 + %iv.next = add i32 %iv, 1 + %ec = icmp slt i32 %iv.next, %shr + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + +define void @ult_no_umax_needed(i64 %n, ptr %dst) { +; CHECK-LABEL: @ult_no_umax_needed( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[N_TRUNC:%.*]] = trunc i64 [[N:%.*]] to i32 +; CHECK-NEXT: [[ADD_1:%.*]] = add i32 [[N_TRUNC]], 1 +; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[ADD_1]], 1 +; CHECK-NEXT: [[PRE:%.*]] = icmp ult i32 [[ADD_1]], 8 +; CHECK-NEXT: br i1 [[PRE]], label [[EXIT:%.*]], label [[LOOP_PREHEADER:%.*]] +; CHECK: loop.preheader: +; CHECK-NEXT: [[UMAX:%.*]] = call i32 @llvm.umax.i32(i32 [[SHR]], i32 1) +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], [[LOOP]] ], [ 0, [[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i32 [[IV]] +; CHECK-NEXT: store i8 0, ptr [[GEP]], align 1 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i32 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[UMAX]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]] +; CHECK: exit.loopexit: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + %n.trunc = trunc i64 %n to i32 + %add.1 = add i32 %n.trunc, 1 + %shr = lshr i32 %add.1, 1 + %pre = icmp ult i32 %add.1, 8 + br i1 %pre, label %exit, label %loop + +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] + %gep = getelementptr inbounds i8, ptr %dst, i32 %iv + store i8 0, ptr %gep, align 1 + %iv.next = add i32 %iv, 1 + %ec = icmp ult i32 %iv.next, %shr + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + !0 = !{i32 1, i32 2147483648} diff --git a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll index 824e23fcf3e6e..c1d8395965577 100644 --- a/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll +++ b/llvm/test/Transforms/LoopUnroll/peel-last-iteration-with-guards.ll @@ -201,6 +201,78 @@ loop.latch: %ec = icmp eq i32 %iv.next, %n br i1 %ec, label %exit, label %loop.header +exit: + ret void +} + +define void @test_peel_guard_sub_1_btc(i32 %n) { +; CHECK-LABEL: define void @test_peel_guard_sub_1_btc( +; CHECK-SAME: i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[SUB:%.*]] = add i32 [[N]], -1 +; CHECK-NEXT: [[PRE:%.*]] = icmp eq i32 [[SUB]], 0 +; CHECK-NEXT: br i1 [[PRE]], label %[[EXIT:.*]], label %[[LOOP_HEADER_PREHEADER:.*]] +; CHECK: [[LOOP_HEADER_PREHEADER]]: +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N]], -2 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne i32 [[TMP0]], 0 +; CHECK-NEXT: br i1 [[TMP1]], label %[[LOOP_HEADER_PREHEADER_SPLIT:.*]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN:.*]] +; CHECK: [[LOOP_HEADER_PREHEADER_SPLIT]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 1, %[[LOOP_HEADER_PREHEADER_SPLIT]] ] +; CHECK-NEXT: br i1 false, label %[[LOOP_LATCH]], label %[[THEN:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: [[CALL136:%.*]] = load volatile ptr, ptr null, align 4294967296 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = sub i32 [[N]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[TMP2]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT:.*]], label %[[LOOP_HEADER]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: [[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]]: +; CHECK-NEXT: [[DOTPH:%.*]] = phi i32 [ [[IV_NEXT]], %[[LOOP_LATCH]] ] +; CHECK-NEXT: br label %[[EXIT_LOOPEXIT_PEEL_BEGIN]] +; CHECK: [[EXIT_LOOPEXIT_PEEL_BEGIN]]: +; CHECK-NEXT: [[TMP3:%.*]] = phi i32 [ 1, %[[LOOP_HEADER_PREHEADER]] ], [ [[DOTPH]], %[[EXIT_LOOPEXIT_PEEL_BEGIN_LOOPEXIT]] ] +; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL:.*]] +; CHECK: [[LOOP_HEADER_PEEL]]: +; CHECK-NEXT: [[CMP115_PEEL:%.*]] = icmp eq i32 [[TMP3]], [[SUB]] +; CHECK-NEXT: br i1 [[CMP115_PEEL]], label %[[LOOP_LATCH_PEEL:.*]], label %[[THEN_PEEL:.*]] +; CHECK: [[THEN_PEEL]]: +; CHECK-NEXT: [[CALL136_PEEL:%.*]] = load volatile ptr, ptr null, align 4294967296 +; CHECK-NEXT: br label %[[LOOP_LATCH_PEEL]] +; CHECK: [[LOOP_LATCH_PEEL]]: +; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add nuw i32 [[TMP3]], 1 +; CHECK-NEXT: [[EC_PEEL:%.*]] = icmp eq i32 [[IV_NEXT_PEEL]], [[N]] +; CHECK-NEXT: br i1 [[EC_PEEL]], label %[[EXIT_LOOPEXIT_PEEL_NEXT:.*]], label %[[EXIT_LOOPEXIT_PEEL_NEXT]] +; CHECK: [[EXIT_LOOPEXIT_PEEL_NEXT]]: +; CHECK-NEXT: br label %[[LOOP_HEADER_PEEL_NEXT:.*]] +; CHECK: [[LOOP_HEADER_PEEL_NEXT]]: +; CHECK-NEXT: br label %[[EXIT_LOOPEXIT:.*]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + %sub = add i32 %n, -1 + %pre = icmp eq i32 %sub, 0 + br i1 %pre, label %exit, label %loop.header + +loop.header: ; preds = %loop.latch, %entry + %iv = phi i32 [ %iv.next, %loop.latch ], [ 1, %entry ] + %cmp115 = icmp eq i32 %iv, %sub + br i1 %cmp115, label %loop.latch, label %then + +then: + %call136 = load volatile ptr, ptr null, align 4294967296 + br label %loop.latch + +loop.latch: + %iv.next = add nuw i32 %iv, 1 + %ec = icmp eq i32 %iv.next, %n + br i1 %ec, label %exit, label %loop.header + exit: ret void } @@ -208,4 +280,5 @@ exit: ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]]} ; CHECK: [[META1]] = !{!"llvm.loop.peeled.count", i32 1} ; CHECK: [[LOOP2]] = distinct !{[[LOOP2]], [[META1]]} +; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]} ;. diff --git a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll index b640c1911cb0d..99a86c94b23dd 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-checks-difference.ll @@ -434,3 +434,62 @@ loop: exit: ret void } + + +define void @remove_diff_checks_via_guards(i32 %x, i32 %y, ptr %A) { +; CHECK-LABEL: define void @remove_diff_checks_via_guards( +; CHECK-SAME: i32 [[X:%.*]], i32 [[Y:%.*]], ptr [[A:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[OFFSET:%.*]] = sub i32 [[X]], [[Y]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sge i32 [[OFFSET]], 0 +; CHECK-NEXT: br i1 [[CMP]], [[EXIT:label %.*]], label %[[LOOP_PREHEADER:.*]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: [[TMP0:%.*]] = sext i32 [[X]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = add nsw i64 [[TMP0]], 1 +; CHECK-NEXT: [[SMAX2:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP1]], i64 0) +; CHECK-NEXT: [[TMP2:%.*]] = trunc i64 [[SMAX2]] to i32 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw i32 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i32 [[TMP3]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], [[SCALAR_PH:label %.*]], label %[[VECTOR_SCEVCHECK:.*]] +; CHECK: [[VECTOR_SCEVCHECK]]: +; CHECK-NEXT: [[TMP4:%.*]] = sext i32 [[X]] to i64 +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 1 +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP5]], i64 0) +; CHECK-NEXT: [[TMP6:%.*]] = trunc i64 [[SMAX]] to i32 +; CHECK-NEXT: [[TMP7:%.*]] = icmp slt i32 [[TMP6]], 0 +; CHECK-NEXT: [[TMP8:%.*]] = icmp ugt i64 [[SMAX]], 4294967295 +; CHECK-NEXT: [[TMP9:%.*]] = or i1 [[TMP7]], [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = trunc i64 [[SMAX]] to i32 +; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[OFFSET]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = icmp slt i32 [[TMP11]], [[OFFSET]] +; CHECK-NEXT: [[TMP13:%.*]] = icmp ugt i64 [[SMAX]], 4294967295 +; CHECK-NEXT: [[TMP14:%.*]] = or i1 [[TMP12]], [[TMP13]] +; CHECK-NEXT: [[TMP15:%.*]] = or i1 [[TMP9]], [[TMP14]] +; CHECK-NEXT: br i1 [[TMP15]], [[SCALAR_PH]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[TMP16:%.*]] = sext i32 [[OFFSET]] to i64 +; CHECK-NEXT: [[TMP17:%.*]] = shl nsw i64 [[TMP16]], 2 +; CHECK-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP17]], 16 +; CHECK-NEXT: br i1 [[DIFF_CHECK]], [[SCALAR_PH]], [[VECTOR_PH1:label %.*]] +; +entry: + %offset = sub i32 %x, %y + %cmp = icmp sge i32 %offset, 0 + br i1 %cmp, label %exit, label %loop + +loop: + %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] + %iv.ext = sext i32 %iv to i64 + %gep.A = getelementptr i32, ptr %A, i64 %iv.ext + %l = load i32, ptr %gep.A, align 1 + %iv.offset = add i32 %iv, %offset + %iv.offset.ext = sext i32 %iv.offset to i64 + %gep.A.offset = getelementptr i32, ptr %A, i64 %iv.offset.ext + store i32 %l, ptr %gep.A.offset, align 1 + %iv.next = add i32 %iv, 1 + %ec = icmp sgt i32 %iv, %x + br i1 %ec, label %exit, label %loop + +exit: + ret void +} From 2e0796d4c5fa186c7b2169cb704746846fa72ea3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 22 Sep 2025 11:31:03 +0100 Subject: [PATCH 37/60] [SCEV] Add additional test with guards for 3-op AddRec. Add additional tests for using information from loop guards in SCEV. (cherry picked from commit e8e678cfa2aa8322942ea8719f9e06b316096156) --- ...ge-taken-count-guard-info-apply-to-adds.ll | 34 ++++++++++++ .../Transforms/LoopIdiom/add-nsw-zext-fold.ll | 55 ++++++++++++++++++- 2 files changed, 88 insertions(+), 1 deletion(-) diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll index 8df4b52757753..6b2c78cebc44a 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info-apply-to-adds.ll @@ -56,3 +56,37 @@ loop: exit: ret void } + +declare void @use(ptr) + +define i32 @test_3_op_add(i32 %x, i32 %y, ptr %A) { +; CHECK-LABEL: 'test_3_op_add' +; CHECK-NEXT: Determining loop execution counts for: @test_3_op_add +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (zext i32 (1 + (-1 * %x) + %y) to i64)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2147483647 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (zext i32 (1 + (-1 * %x) + %y) to i64)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %pre.0 = icmp ugt i32 %x, 0 + br i1 %pre.0, label %then, label %exit + +then: + %y.sub.x = sub i32 %y, %x + %pre.1 = icmp slt i32 %y.sub.x, 0 + %add.1 = add i32 %y.sub.x, 1 + %add.ext = zext i32 %add.1 to i64 + br i1 %pre.1, label %exit, label %loop + +loop: + %iv = phi i64 [ %iv.next, %loop ], [ 0, %then ] + %and = and i64 %iv, 1 + %gep = getelementptr i8, ptr %A, i64 %and + call void @use(ptr %gep) + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %add.ext + br i1 %ec, label %exit, label %loop + +exit: + ret i32 0 +} diff --git a/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll b/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll index fc2b5571250ac..bc1543d8361a7 100644 --- a/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll +++ b/llvm/test/Transforms/LoopIdiom/add-nsw-zext-fold.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 ; RUN: opt -p loop-idiom -S %s | FileCheck %s define void @fold_add_zext_to_sext(ptr %dst, i1 %start) { @@ -40,3 +40,56 @@ loop: exit: ret void } + +declare i16 @get() + +define void @test_memset_size_can_use_info_from_guards(i32 %x, ptr %dst) { +; CHECK-LABEL: define void @test_memset_size_can_use_info_from_guards( +; CHECK-SAME: i32 [[X:%.*]], ptr [[DST:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP1:.*]] +; CHECK: [[LOOP1_LOOPEXIT:.*]]: +; CHECK-NEXT: br label %[[LOOP1_BACKEDGE:.*]] +; CHECK: [[LOOP1]]: +; CHECK-NEXT: [[P:%.*]] = phi i32 [ 0, %[[ENTRY]] ], [ [[X]], %[[LOOP1_BACKEDGE]] ] +; CHECK-NEXT: [[L:%.*]] = call i16 @get() +; CHECK-NEXT: [[L_EXT:%.*]] = zext i16 [[L]] to i32 +; CHECK-NEXT: [[SUB:%.*]] = sub i32 [[L_EXT]], [[P]] +; CHECK-NEXT: [[SHR:%.*]] = lshr i32 [[SUB]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp ult i32 [[SUB]], 2 +; CHECK-NEXT: br i1 [[EC]], label %[[LOOP1_BACKEDGE]], label %[[LOOP2_PREHEADER:.*]] +; CHECK: [[LOOP1_BACKEDGE]]: +; CHECK-NEXT: br label %[[LOOP1]] +; CHECK: [[LOOP2_PREHEADER]]: +; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[SUB]] to i64 +; CHECK-NEXT: [[TMP1:%.*]] = lshr i64 [[TMP0]], 1 +; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP1]], i64 1) +; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 1 [[DST]], i8 0, i64 [[UMAX]], i1 false) +; CHECK-NEXT: br label %[[LOOP2:.*]] +; CHECK: [[LOOP2]]: +; CHECK-NEXT: [[IV_2:%.*]] = phi i32 [ [[IV_2_NEXT:%.*]], %[[LOOP2]] ], [ 0, %[[LOOP2_PREHEADER]] ] +; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr i8, ptr [[DST]], i32 [[IV_2]] +; CHECK-NEXT: [[IV_2_NEXT]] = add i32 [[IV_2]], 1 +; CHECK-NEXT: [[EC_2:%.*]] = icmp ult i32 [[IV_2_NEXT]], [[SHR]] +; CHECK-NEXT: br i1 [[EC_2]], label %[[LOOP2]], label %[[LOOP1_LOOPEXIT]] +; +entry: + br label %loop1 + +loop1: + %p = phi i32 [ 0, %entry ], [ %x, %loop1 ], [ %x, %loop2 ] + %l = call i16 @get() + %l.ext = zext i16 %l to i32 + %sub = sub i32 %l.ext, %p + %shr = lshr i32 %sub, 1 + %ec = icmp ult i32 %sub, 2 + br i1 %ec, label %loop1, label %loop2 + +loop2: + %iv.2 = phi i32 [ 0, %loop1 ], [ %iv.2.next, %loop2 ] + %gep.dst = getelementptr i8, ptr %dst, i32 %iv.2 + store i8 0, ptr %gep.dst, align 1 + %iv.2.next = add i32 %iv.2, 1 + %ec.2 = icmp ult i32 %iv.2.next, %shr + br i1 %ec.2, label %loop2, label %loop1 +} From 14064d5ab5d9f66a4a1f04d61820db8989ec73ea Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 22 Sep 2025 20:58:52 +0100 Subject: [PATCH 38/60] [LV] Add test showing missed optimization due to missing info from guard Add test for SCEVUMaxExpr handling in https://github.com/llvm/llvm-project/pull/160012. (cherry picked from commit 129c6836b772988b77b60683f8b1e6d65ddc1ce4) --- .../LoopVectorize/single_early_exit.ll | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll index ddf238ffba4b4..e22cb9c68ede8 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll @@ -578,6 +578,73 @@ exit: ret ptr %res } +define i64 @loop_guards_needed_to_prove_deref_multiple(i32 %x, i1 %c, ptr dereferenceable(1024) %src) { +; CHECK-LABEL: define i64 @loop_guards_needed_to_prove_deref_multiple( +; CHECK-SAME: i32 [[X:%.*]], i1 [[C:%.*]], ptr dereferenceable(1024) [[SRC:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_AND:%.*]] = and i32 [[X]], -2 +; CHECK-NEXT: [[PRE_0:%.*]] = icmp eq i32 [[X]], 0 +; CHECK-NEXT: br i1 [[PRE_0]], label [[THEN:%.*]], label [[EXIT:%.*]] +; CHECK: then: +; CHECK-NEXT: [[SEL:%.*]] = select i1 [[C]], i32 [[X_AND]], i32 0 +; CHECK-NEXT: [[PRE_1:%.*]] = icmp ugt i32 [[SEL]], 1024 +; CHECK-NEXT: br i1 [[PRE_1]], label [[EXIT]], label [[PH:%.*]] +; CHECK: ph: +; CHECK-NEXT: [[PRE_2:%.*]] = icmp ne i32 [[SEL]], 0 +; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_2]]) +; CHECK-NEXT: [[N:%.*]] = add i32 [[SEL]], -1 +; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[PH]] ] +; CHECK-NEXT: [[GEP_SRC_I:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV]] +; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC_I]], align 1 +; CHECK-NEXT: [[C_1:%.*]] = icmp eq i8 [[L]], 0 +; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N_EXT]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]] +; CHECK: exit.loopexit: +; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV]], [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ] +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: [[RES:%.*]] = phi i64 [ -1, [[ENTRY:%.*]] ], [ -2, [[THEN]] ], [ [[RES_PH]], [[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret i64 [[RES]] +; +entry: + %x.and = and i32 %x, -2 + %pre.0 = icmp eq i32 %x, 0 + br i1 %pre.0, label %then, label %exit + +then: + %sel = select i1 %c, i32 %x.and, i32 0 + %pre.1 = icmp ugt i32 %sel, 1024 + br i1 %pre.1, label %exit, label %ph + +ph: + %pre.2 = icmp ne i32 %sel, 0 + call void @llvm.assume(i1 %pre.2) + %n = add i32 %sel, -1 + %n.ext = zext i32 %n to i64 + br label %loop.header + +loop.header: + %iv = phi i64 [ %iv.next, %loop.latch ], [ 0, %ph ] + %gep.src.i = getelementptr i8, ptr %src, i64 %iv + %l = load i8, ptr %gep.src.i, align 1 + %c.1 = icmp eq i8 %l, 0 + br i1 %c.1, label %exit, label %loop.latch + +loop.latch: + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, %n.ext + br i1 %ec, label %exit, label %loop.header + +exit: + %res = phi i64 [ -1, %entry ], [ -2, %then ], [ 0, %loop.latch ], [ %iv, %loop.header ] + ret i64 %res +} ;. ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} ; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} From 921ddad91299ccea73f03bdc911eb42f36482662 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 26 Sep 2025 13:12:51 +0100 Subject: [PATCH 39/60] [SCEV] Add tests for computing trip counts with align assumptions. (cherry picked from commit 9be276ec75c087595ebb62fe11b35c1a90371a49) --- .../trip-multiple-guard-info.ll | 159 ++++++++++++++++++ 1 file changed, 159 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll index bf140c7fa216a..b1fe7b1b2b7ee 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll @@ -574,5 +574,164 @@ exit: ret void } +define void @test_ptr_aligned_by_2_and_4_via_assumption(ptr %start, ptr %end) { +; CHECK-LABEL: 'test_ptr_aligned_by_2_and_4_via_assumption' +; CHECK-NEXT: Classifying expressions for: @test_ptr_aligned_by_2_and_4_via_assumption +; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_ptr_aligned_by_2_and_4_via_assumption +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 2) ] + call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 4) ] + br label %loop + +loop: + %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] + store ptr %iv, ptr %iv + %iv.next = getelementptr i8, ptr %iv, i64 4 + %ec = icmp ne ptr %iv.next, %end + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + +define void @test_ptrs_aligned_by_4_via_assumption(ptr %start, ptr %end) { +; CHECK-LABEL: 'test_ptrs_aligned_by_4_via_assumption' +; CHECK-NEXT: Classifying expressions for: @test_ptrs_aligned_by_4_via_assumption +; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_ptrs_aligned_by_4_via_assumption +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 4) ] + call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 4) ] + br label %loop + +loop: + %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] + store ptr %iv, ptr %iv + %iv.next = getelementptr i8, ptr %iv, i64 4 + %ec = icmp ne ptr %iv.next, %end + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + +define void @test_ptrs_aligned_by_8_via_assumption(ptr %start, ptr %end) { +; CHECK-LABEL: 'test_ptrs_aligned_by_8_via_assumption' +; CHECK-NEXT: Classifying expressions for: @test_ptrs_aligned_by_8_via_assumption +; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_ptrs_aligned_by_8_via_assumption +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 8) ] + call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 8) ] + br label %loop + +loop: + %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] + store ptr %iv, ptr %iv + %iv.next = getelementptr i8, ptr %iv, i64 4 + %ec = icmp ne ptr %iv.next, %end + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + +declare i1 @cond() + +define void @test_ptr_aligned_by_4_via_assumption_multiple_loop_predecessors(ptr %start, ptr %end) { +; CHECK-LABEL: 'test_ptr_aligned_by_4_via_assumption_multiple_loop_predecessors' +; CHECK-NEXT: Classifying expressions for: @test_ptr_aligned_by_4_via_assumption_multiple_loop_predecessors +; CHECK-NEXT: %c = call i1 @cond() +; CHECK-NEXT: --> %c U: full-set S: full-set +; CHECK-NEXT: %iv = phi ptr [ %start, %then ], [ %start, %else ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_ptr_aligned_by_4_via_assumption_multiple_loop_predecessors +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 2) ] + call void @llvm.assume(i1 true) [ "align"(ptr %end, i64 4) ] + %c = call i1 @cond() + br i1 %c, label %then, label %else + +then: + br label %loop + +else: + br label %loop + +loop: + %iv = phi ptr [ %start, %then] , [ %start, %else ], [ %iv.next, %loop ] + store ptr %iv, ptr %iv + %iv.next = getelementptr i8, ptr %iv, i64 4 + %ec = icmp ne ptr %iv.next, %end + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + declare void @llvm.assume(i1) declare void @llvm.experimental.guard(i1, ...) From 037fffd450dfacb29d0592dc55d938f213cc00c9 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 8 Oct 2025 13:04:30 +0100 Subject: [PATCH 40/60] [SCEV] Pass loop pred branch as context instruction to getMinTrailingZ. (#160941) When computing the backedge taken count, we know that the expression must be valid just before we enter the loop. Using the terminator of the loop predecessor as context instruction for getConstantMultiple, getMinTrailingZeros allows using information from things like alignment assumptions. When a context instruction is used, the result is not cached, as it is only valid at the specific context instruction. Compile-time looks neutral: http://llvm-compile-time-tracker.com/compare.php?from=9be276ec75c087595ebb62fe11b35c1a90371a49&to=745980f5e1c8094ea1293cd145d0ef1390f03029&stat=instructions:u No impact on llvm-opt-benchmark (https://github.com/dtcxzyw/llvm-opt-benchmark/pull/2867), but leads to additonal unrolling in ~90 files across a C/C++ based corpus including LLVM on AArch64 using libc++ (which emits alignment assumptions for things like std::vector::begin). PR: https://github.com/llvm/llvm-project/pull/160941 (cherry picked from commit c7fbe388938b2c9ee78a3160fedebd9bebe5d20d) --- llvm/include/llvm/Analysis/ScalarEvolution.h | 16 ++++-- llvm/lib/Analysis/ScalarEvolution.cpp | 56 +++++++++++-------- .../trip-multiple-guard-info.ll | 40 ++++--------- 3 files changed, 57 insertions(+), 55 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index b83fc3b72abf3..faebf07afc107 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1000,10 +1000,14 @@ class ScalarEvolution { /// (at every loop iteration). It is, at the same time, the minimum number /// of times S is divisible by 2. For example, given {4,+,8} it returns 2. /// If S is guaranteed to be 0, it returns the bitwidth of S. - LLVM_ABI uint32_t getMinTrailingZeros(const SCEV *S); + /// If \p CtxI is not nullptr, return a constant multiple valid at \p CtxI. + LLVM_ABI uint32_t getMinTrailingZeros(const SCEV *S, + const Instruction *CtxI = nullptr); - /// Returns the max constant multiple of S. - LLVM_ABI APInt getConstantMultiple(const SCEV *S); + /// Returns the max constant multiple of S. If \p CtxI is not nullptr, return + /// a constant multiple valid at \p CtxI. + LLVM_ABI APInt getConstantMultiple(const SCEV *S, + const Instruction *CtxI = nullptr); // Returns the max constant multiple of S. If S is exactly 0, return 1. LLVM_ABI APInt getNonZeroConstantMultiple(const SCEV *S); @@ -1525,8 +1529,10 @@ class ScalarEvolution { /// Return the Value set from which the SCEV expr is generated. ArrayRef getSCEVValues(const SCEV *S); - /// Private helper method for the getConstantMultiple method. - APInt getConstantMultipleImpl(const SCEV *S); + /// Private helper method for the getConstantMultiple method. If \p CtxI is + /// not nullptr, return a constant multiple valid at \p CtxI. + APInt getConstantMultipleImpl(const SCEV *S, + const Instruction *Ctx = nullptr); /// Information about the number of times a particular loop exit may be /// reached before exiting the loop. diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 0b13dfb1617f1..c03c759ba2593 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -6344,19 +6344,20 @@ const SCEV *ScalarEvolution::createNodeForGEP(GEPOperator *GEP) { return getGEPExpr(GEP, IndexExprs); } -APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) { +APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S, + const Instruction *CtxI) { uint64_t BitWidth = getTypeSizeInBits(S->getType()); auto GetShiftedByZeros = [BitWidth](uint32_t TrailingZeros) { return TrailingZeros >= BitWidth ? APInt::getZero(BitWidth) : APInt::getOneBitSet(BitWidth, TrailingZeros); }; - auto GetGCDMultiple = [this](const SCEVNAryExpr *N) { + auto GetGCDMultiple = [this, CtxI](const SCEVNAryExpr *N) { // The result is GCD of all operands results. - APInt Res = getConstantMultiple(N->getOperand(0)); + APInt Res = getConstantMultiple(N->getOperand(0), CtxI); for (unsigned I = 1, E = N->getNumOperands(); I < E && Res != 1; ++I) Res = APIntOps::GreatestCommonDivisor( - Res, getConstantMultiple(N->getOperand(I))); + Res, getConstantMultiple(N->getOperand(I), CtxI)); return Res; }; @@ -6364,33 +6365,33 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) { case scConstant: return cast(S)->getAPInt(); case scPtrToInt: - return getConstantMultiple(cast(S)->getOperand()); + return getConstantMultiple(cast(S)->getOperand(), CtxI); case scUDivExpr: case scVScale: return APInt(BitWidth, 1); case scTruncate: { // Only multiples that are a power of 2 will hold after truncation. const SCEVTruncateExpr *T = cast(S); - uint32_t TZ = getMinTrailingZeros(T->getOperand()); + uint32_t TZ = getMinTrailingZeros(T->getOperand(), CtxI); return GetShiftedByZeros(TZ); } case scZeroExtend: { const SCEVZeroExtendExpr *Z = cast(S); - return getConstantMultiple(Z->getOperand()).zext(BitWidth); + return getConstantMultiple(Z->getOperand(), CtxI).zext(BitWidth); } case scSignExtend: { // Only multiples that are a power of 2 will hold after sext. const SCEVSignExtendExpr *E = cast(S); - uint32_t TZ = getMinTrailingZeros(E->getOperand()); + uint32_t TZ = getMinTrailingZeros(E->getOperand(), CtxI); return GetShiftedByZeros(TZ); } case scMulExpr: { const SCEVMulExpr *M = cast(S); if (M->hasNoUnsignedWrap()) { // The result is the product of all operand results. - APInt Res = getConstantMultiple(M->getOperand(0)); + APInt Res = getConstantMultiple(M->getOperand(0), CtxI); for (const SCEV *Operand : M->operands().drop_front()) - Res = Res * getConstantMultiple(Operand); + Res = Res * getConstantMultiple(Operand, CtxI); return Res; } @@ -6398,7 +6399,7 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) { // sum of trailing zeros for all its operands. uint32_t TZ = 0; for (const SCEV *Operand : M->operands()) - TZ += getMinTrailingZeros(Operand); + TZ += getMinTrailingZeros(Operand, CtxI); return GetShiftedByZeros(TZ); } case scAddExpr: @@ -6407,9 +6408,9 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) { if (N->hasNoUnsignedWrap()) return GetGCDMultiple(N); // Find the trailing bits, which is the minimum of its operands. - uint32_t TZ = getMinTrailingZeros(N->getOperand(0)); + uint32_t TZ = getMinTrailingZeros(N->getOperand(0), CtxI); for (const SCEV *Operand : N->operands().drop_front()) - TZ = std::min(TZ, getMinTrailingZeros(Operand)); + TZ = std::min(TZ, getMinTrailingZeros(Operand, CtxI)); return GetShiftedByZeros(TZ); } case scUMaxExpr: @@ -6422,7 +6423,7 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) { // ask ValueTracking for known bits const SCEVUnknown *U = cast(S); unsigned Known = - computeKnownBits(U->getValue(), getDataLayout(), &AC, nullptr, &DT) + computeKnownBits(U->getValue(), getDataLayout(), &AC, CtxI, &DT) .countMinTrailingZeros(); return GetShiftedByZeros(Known); } @@ -6432,12 +6433,18 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S) { llvm_unreachable("Unknown SCEV kind!"); } -APInt ScalarEvolution::getConstantMultiple(const SCEV *S) { +APInt ScalarEvolution::getConstantMultiple(const SCEV *S, + const Instruction *CtxI) { + // Skip looking up and updating the cache if there is a context instruction, + // as the result will only be valid in the specified context. + if (CtxI) + return getConstantMultipleImpl(S, CtxI); + auto I = ConstantMultipleCache.find(S); if (I != ConstantMultipleCache.end()) return I->second; - APInt Result = getConstantMultipleImpl(S); + APInt Result = getConstantMultipleImpl(S, CtxI); auto InsertPair = ConstantMultipleCache.insert({S, Result}); assert(InsertPair.second && "Should insert a new key"); return InsertPair.first->second; @@ -6448,8 +6455,9 @@ APInt ScalarEvolution::getNonZeroConstantMultiple(const SCEV *S) { return Multiple == 0 ? APInt(Multiple.getBitWidth(), 1) : Multiple; } -uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S) { - return std::min(getConstantMultiple(S).countTrailingZeros(), +uint32_t ScalarEvolution::getMinTrailingZeros(const SCEV *S, + const Instruction *CtxI) { + return std::min(getConstantMultiple(S, CtxI).countTrailingZeros(), (unsigned)getTypeSizeInBits(S->getType())); } @@ -10228,8 +10236,7 @@ const SCEV *ScalarEvolution::stripInjectiveFunctions(const SCEV *S) const { static const SCEV * SolveLinEquationWithOverflow(const APInt &A, const SCEV *B, SmallVectorImpl *Predicates, - - ScalarEvolution &SE) { + ScalarEvolution &SE, const Loop *L) { uint32_t BW = A.getBitWidth(); assert(BW == SE.getTypeSizeInBits(B->getType())); assert(A != 0 && "A must be non-zero."); @@ -10245,7 +10252,12 @@ SolveLinEquationWithOverflow(const APInt &A, const SCEV *B, // // B is divisible by D if and only if the multiplicity of prime factor 2 for B // is not less than multiplicity of this prime factor for D. - if (SE.getMinTrailingZeros(B) < Mult2) { + unsigned MinTZ = SE.getMinTrailingZeros(B); + // Try again with the terminator of the loop predecessor for context-specific + // result, if MinTZ s too small. + if (MinTZ < Mult2 && L->getLoopPredecessor()) + MinTZ = SE.getMinTrailingZeros(B, L->getLoopPredecessor()->getTerminator()); + if (MinTZ < Mult2) { // Check if we can prove there's no remainder using URem. const SCEV *URem = SE.getURemExpr(B, SE.getConstant(APInt::getOneBitSet(BW, Mult2))); @@ -10693,7 +10705,7 @@ ScalarEvolution::ExitLimit ScalarEvolution::howFarToZero(const SCEV *V, return getCouldNotCompute(); const SCEV *E = SolveLinEquationWithOverflow( StepC->getAPInt(), getNegativeSCEV(Start), - AllowPredicates ? &Predicates : nullptr, *this); + AllowPredicates ? &Predicates : nullptr, *this, L); const SCEV *M = E; if (E != getCouldNotCompute()) { diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll index b1fe7b1b2b7ee..7ba422da79ad8 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll @@ -615,22 +615,14 @@ define void @test_ptrs_aligned_by_4_via_assumption(ptr %start, ptr %end) { ; CHECK-LABEL: 'test_ptrs_aligned_by_4_via_assumption' ; CHECK-NEXT: Classifying expressions for: @test_ptrs_aligned_by_4_via_assumption ; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 -; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_ptrs_aligned_by_4_via_assumption -; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 -; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 -; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 -; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 4) ] @@ -652,22 +644,14 @@ define void @test_ptrs_aligned_by_8_via_assumption(ptr %start, ptr %end) { ; CHECK-LABEL: 'test_ptrs_aligned_by_8_via_assumption' ; CHECK-NEXT: Classifying expressions for: @test_ptrs_aligned_by_8_via_assumption ; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 -; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_ptrs_aligned_by_8_via_assumption -; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. -; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. -; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 -; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 -; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 -; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) -; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 8) ] From 8d201e51879845c27a6b3e2f429d3838053cb7b5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 9 Oct 2025 10:51:36 +0100 Subject: [PATCH 41/60] [SCEV] Use getConstantMultiple in to get divisibility info from guards. (#162617) Simplify and generalize the code to get a common constant multiple for expressions when collecting guards, replacing the manual implementation. Split off from https://github.com/llvm/llvm-project/pull/160012. PR: https://github.com/llvm/llvm-project/pull/162617 (cherry picked from commit 6d905e41bc82f1a6ae88484a700099fbe26460b6) --- llvm/lib/Analysis/ScalarEvolution.cpp | 46 ++-------------- .../ScalarEvolution/trip-count-minmax.ll | 4 +- .../LoopVectorize/single_early_exit.ll | 52 ++++++++++++++----- 3 files changed, 43 insertions(+), 59 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index c03c759ba2593..376608d97d1df 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15683,51 +15683,11 @@ void ScalarEvolution::LoopGuards::collectFromBlock( return RewriteMap.lookup_or(S, S); }; - // Check for the SCEV expression (A /u B) * B while B is a constant, inside - // \p Expr. The check is done recuresively on \p Expr, which is assumed to - // be a composition of Min/Max SCEVs. Return whether the SCEV expression (A - // /u B) * B was found, and return the divisor B in \p DividesBy. For - // example, if Expr = umin (umax ((A /u 8) * 8, 16), 64), return true since - // (A /u 8) * 8 matched the pattern, and return the constant SCEV 8 in \p - // DividesBy. - std::function HasDivisibiltyInfo = - [&](const SCEV *Expr, const SCEV *&DividesBy) { - if (auto *Mul = dyn_cast(Expr)) { - if (Mul->getNumOperands() != 2) - return false; - auto *MulLHS = Mul->getOperand(0); - auto *MulRHS = Mul->getOperand(1); - if (isa(MulLHS)) - std::swap(MulLHS, MulRHS); - if (auto *Div = dyn_cast(MulLHS)) - if (Div->getOperand(1) == MulRHS) { - DividesBy = MulRHS; - return true; - } - } - if (auto *MinMax = dyn_cast(Expr)) - return HasDivisibiltyInfo(MinMax->getOperand(0), DividesBy) || - HasDivisibiltyInfo(MinMax->getOperand(1), DividesBy); - return false; - }; - - // Return true if Expr known to divide by \p DividesBy. - std::function IsKnownToDivideBy = - [&](const SCEV *Expr, const SCEV *DividesBy) { - if (SE.getURemExpr(Expr, DividesBy)->isZero()) - return true; - if (auto *MinMax = dyn_cast(Expr)) - return IsKnownToDivideBy(MinMax->getOperand(0), DividesBy) && - IsKnownToDivideBy(MinMax->getOperand(1), DividesBy); - return false; - }; - const SCEV *RewrittenLHS = GetMaybeRewritten(LHS); const SCEV *DividesBy = nullptr; - if (HasDivisibiltyInfo(RewrittenLHS, DividesBy)) - // Check that the whole expression is divided by DividesBy - DividesBy = - IsKnownToDivideBy(RewrittenLHS, DividesBy) ? DividesBy : nullptr; + const APInt &Multiple = SE.getConstantMultiple(RewrittenLHS); + if (!Multiple.isOne()) + DividesBy = SE.getConstant(Multiple); // Collect rewrites for LHS and its transitive operands based on the // condition. diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll index 8d091a00ed4b9..d38010403dad7 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count-minmax.ll @@ -61,7 +61,7 @@ define void @umin(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) umin (4 * %b))) -; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; ; void umin(unsigned a, unsigned b) { ; a *= 2; @@ -157,7 +157,7 @@ define void @smin(i32 noundef %a, i32 noundef %b) { ; CHECK-NEXT: Loop %for.body: backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) ; CHECK-NEXT: Loop %for.body: constant max backedge-taken count is i32 2147483646 ; CHECK-NEXT: Loop %for.body: symbolic max backedge-taken count is (-1 + ((2 * %a) smin (4 * %b))) -; CHECK-NEXT: Loop %for.body: Trip multiple is 1 +; CHECK-NEXT: Loop %for.body: Trip multiple is 2 ; ; void smin(signed a, signed b) { ; a *= 2; diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll index e22cb9c68ede8..7781db64df255 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll @@ -594,19 +594,50 @@ define i64 @loop_guards_needed_to_prove_deref_multiple(i32 %x, i1 %c, ptr derefe ; CHECK-NEXT: call void @llvm.assume(i1 [[PRE_2]]) ; CHECK-NEXT: [[N:%.*]] = add i32 [[SEL]], -1 ; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[SEL]], -2 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 2 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP2]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], 4 +; CHECK-NEXT: [[IV_NEXT:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] ; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_HEADER]] ] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]] +; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IV_NEXT]] +; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] +; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[LOOP_LATCH:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP2]], [[IV_NEXT]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP4]], i1 true) +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: br label [[EXIT_LOOPEXIT]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT]], [[LOOP_LATCH]] ], [ 0, [[PH]] ] +; CHECK-NEXT: br label [[LOOP_HEADER1:%.*]] ; CHECK: loop.header: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ 0, [[PH]] ] -; CHECK-NEXT: [[GEP_SRC_I:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV]] +; CHECK-NEXT: [[IV1:%.*]] = phi i64 [ [[IV_NEXT1:%.*]], [[LOOP_LATCH1:%.*]] ], [ [[IV]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[GEP_SRC_I:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[IV1]] ; CHECK-NEXT: [[L:%.*]] = load i8, ptr [[GEP_SRC_I]], align 1 ; CHECK-NEXT: [[C_1:%.*]] = icmp eq i8 [[L]], 0 -; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT:%.*]], label [[LOOP_LATCH]] +; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT]], label [[LOOP_LATCH1]] ; CHECK: loop.latch: -; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 -; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], [[N_EXT]] -; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]] +; CHECK-NEXT: [[IV_NEXT1]] = add i64 [[IV1]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N_EXT]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER1]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: exit.loopexit: -; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV]], [[LOOP_HEADER]] ], [ 0, [[LOOP_LATCH]] ] +; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV1]], [[LOOP_HEADER1]] ], [ 0, [[LOOP_LATCH1]] ], [ 0, [[LOOP_LATCH]] ], [ [[TMP10]], [[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: br label [[EXIT]] ; CHECK: exit: ; CHECK-NEXT: [[RES:%.*]] = phi i64 [ -1, [[ENTRY:%.*]] ], [ -2, [[THEN]] ], [ [[RES_PH]], [[EXIT_LOOPEXIT]] ] @@ -653,11 +684,4 @@ exit: ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} -; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]} -; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} -; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} -; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} -; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} -; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} -; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} ;. From b0ea5358305806e023c998d9db5def439471f0b3 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 12 Oct 2025 19:47:03 +0100 Subject: [PATCH 42/60] [SCEV] Use APInt for DividesBy when collecting loop guard info (NFC). (#163017) Follow-up as suggested in https://github.com/llvm/llvm-project/pull/162617. Just use an APInt for DividesBy, as the existing code already operates on APInt and thus handles the case of DividesBy being 1. PR: https://github.com/llvm/llvm-project/pull/163017 (cherry picked from commit 0d1f2f439d699cd0fe3ec6893b2703c1edf3eea9) --- llvm/lib/Analysis/ScalarEvolution.cpp | 75 ++++++++----------- .../LoopVectorize/single_early_exit.ll | 16 +++- 2 files changed, 45 insertions(+), 46 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 376608d97d1df..481fef9bcb7e6 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15567,47 +15567,34 @@ void ScalarEvolution::LoopGuards::collectFromBlock( return false; }; - // Checks whether Expr is a non-negative constant, and Divisor is a positive - // constant, and returns their APInt in ExprVal and in DivisorVal. - auto GetNonNegExprAndPosDivisor = [&](const SCEV *Expr, const SCEV *Divisor, - APInt &ExprVal, APInt &DivisorVal) { - auto *ConstExpr = dyn_cast(Expr); - auto *ConstDivisor = dyn_cast(Divisor); - if (!ConstExpr || !ConstDivisor) - return false; - ExprVal = ConstExpr->getAPInt(); - DivisorVal = ConstDivisor->getAPInt(); - return ExprVal.isNonNegative() && !DivisorVal.isNonPositive(); - }; - // Return a new SCEV that modifies \p Expr to the closest number divides by - // \p Divisor and greater or equal than Expr. - // For now, only handle constant Expr and Divisor. + // \p Divisor and greater or equal than Expr. For now, only handle constant + // Expr. auto GetNextSCEVDividesByDivisor = [&](const SCEV *Expr, - const SCEV *Divisor) { - APInt ExprVal; - APInt DivisorVal; - if (!GetNonNegExprAndPosDivisor(Expr, Divisor, ExprVal, DivisorVal)) + const APInt &DivisorVal) { + const APInt *ExprVal; + if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || + DivisorVal.isNonPositive()) return Expr; - APInt Rem = ExprVal.urem(DivisorVal); - if (!Rem.isZero()) - // return the SCEV: Expr + Divisor - Expr % Divisor - return SE.getConstant(ExprVal + DivisorVal - Rem); - return Expr; + APInt Rem = ExprVal->urem(DivisorVal); + if (Rem.isZero()) + return Expr; + // return the SCEV: Expr + Divisor - Expr % Divisor + return SE.getConstant(*ExprVal + DivisorVal - Rem); }; // Return a new SCEV that modifies \p Expr to the closest number divides by - // \p Divisor and less or equal than Expr. - // For now, only handle constant Expr and Divisor. + // \p Divisor and less or equal than Expr. For now, only handle constant + // Expr. auto GetPreviousSCEVDividesByDivisor = [&](const SCEV *Expr, - const SCEV *Divisor) { - APInt ExprVal; - APInt DivisorVal; - if (!GetNonNegExprAndPosDivisor(Expr, Divisor, ExprVal, DivisorVal)) + const APInt &DivisorVal) { + const APInt *ExprVal; + if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || + DivisorVal.isNonPositive()) return Expr; - APInt Rem = ExprVal.urem(DivisorVal); + APInt Rem = ExprVal->urem(DivisorVal); // return the SCEV: Expr - Expr % Divisor - return SE.getConstant(ExprVal - Rem); + return SE.getConstant(*ExprVal - Rem); }; // Apply divisibilty by \p Divisor on MinMaxExpr with constant values, @@ -15616,6 +15603,11 @@ void ScalarEvolution::LoopGuards::collectFromBlock( std::function ApplyDivisibiltyOnMinMaxExpr = [&](const SCEV *MinMaxExpr, const SCEV *Divisor) { + auto *ConstDivisor = dyn_cast(Divisor); + if (!ConstDivisor) + return MinMaxExpr; + const APInt &DivisorVal = ConstDivisor->getAPInt(); + const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr; SCEVTypes SCTy; if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS, @@ -15626,8 +15618,8 @@ void ScalarEvolution::LoopGuards::collectFromBlock( assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!"); auto *DivisibleExpr = - IsMin ? GetPreviousSCEVDividesByDivisor(MinMaxLHS, Divisor) - : GetNextSCEVDividesByDivisor(MinMaxLHS, Divisor); + IsMin ? GetPreviousSCEVDividesByDivisor(MinMaxLHS, DivisorVal) + : GetNextSCEVDividesByDivisor(MinMaxLHS, DivisorVal); SmallVector Ops = { ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr}; return SE.getMinMaxExpr(SCTy, Ops); @@ -15684,10 +15676,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( }; const SCEV *RewrittenLHS = GetMaybeRewritten(LHS); - const SCEV *DividesBy = nullptr; - const APInt &Multiple = SE.getConstantMultiple(RewrittenLHS); - if (!Multiple.isOne()) - DividesBy = SE.getConstant(Multiple); + const APInt &DividesBy = SE.getConstantMultiple(RewrittenLHS); // Collect rewrites for LHS and its transitive operands based on the // condition. @@ -15709,21 +15698,21 @@ void ScalarEvolution::LoopGuards::collectFromBlock( [[fallthrough]]; case CmpInst::ICMP_SLT: { RHS = SE.getMinusSCEV(RHS, One); - RHS = DividesBy ? GetPreviousSCEVDividesByDivisor(RHS, DividesBy) : RHS; + RHS = GetPreviousSCEVDividesByDivisor(RHS, DividesBy); break; } case CmpInst::ICMP_UGT: case CmpInst::ICMP_SGT: RHS = SE.getAddExpr(RHS, One); - RHS = DividesBy ? GetNextSCEVDividesByDivisor(RHS, DividesBy) : RHS; + RHS = GetNextSCEVDividesByDivisor(RHS, DividesBy); break; case CmpInst::ICMP_ULE: case CmpInst::ICMP_SLE: - RHS = DividesBy ? GetPreviousSCEVDividesByDivisor(RHS, DividesBy) : RHS; + RHS = GetPreviousSCEVDividesByDivisor(RHS, DividesBy); break; case CmpInst::ICMP_UGE: case CmpInst::ICMP_SGE: - RHS = DividesBy ? GetNextSCEVDividesByDivisor(RHS, DividesBy) : RHS; + RHS = GetNextSCEVDividesByDivisor(RHS, DividesBy); break; default: break; @@ -15777,7 +15766,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( case CmpInst::ICMP_NE: if (match(RHS, m_scev_Zero())) { const SCEV *OneAlignedUp = - DividesBy ? GetNextSCEVDividesByDivisor(One, DividesBy) : One; + GetNextSCEVDividesByDivisor(One, DividesBy); To = SE.getUMaxExpr(FromRewritten, OneAlignedUp); } break; diff --git a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll index 7781db64df255..aa7136b71fae8 100644 --- a/llvm/test/Transforms/LoopVectorize/single_early_exit.ll +++ b/llvm/test/Transforms/LoopVectorize/single_early_exit.ll @@ -606,14 +606,15 @@ define i64 @loop_guards_needed_to_prove_deref_multiple(i32 %x, i1 %c, ptr derefe ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[LOOP_HEADER]] ] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[SRC]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[TMP3]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP11]], align 1 ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i8> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = freeze <4 x i1> [[TMP4]] ; CHECK-NEXT: [[TMP6:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP5]]) ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[IV_NEXT]] ; CHECK-NEXT: [[TMP8:%.*]] = or i1 [[TMP6]], [[TMP7]] -; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_SPLIT:%.*]], label [[LOOP_HEADER]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: middle.split: ; CHECK-NEXT: br i1 [[TMP6]], label [[VECTOR_EARLY_EXIT:%.*]], label [[LOOP_LATCH:%.*]] ; CHECK: middle.block: @@ -635,7 +636,7 @@ define i64 @loop_guards_needed_to_prove_deref_multiple(i32 %x, i1 %c, ptr derefe ; CHECK: loop.latch: ; CHECK-NEXT: [[IV_NEXT1]] = add i64 [[IV1]], 1 ; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV1]], [[N_EXT]] -; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER1]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK-NEXT: br i1 [[EC]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER1]], !llvm.loop [[LOOP15:![0-9]+]] ; CHECK: exit.loopexit: ; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ [[IV1]], [[LOOP_HEADER1]] ], [ 0, [[LOOP_LATCH1]] ], [ 0, [[LOOP_LATCH]] ], [ [[TMP10]], [[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: br label [[EXIT]] @@ -684,4 +685,13 @@ exit: ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} ; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} +; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]]} +; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} +; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} +; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} +; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} +; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} +; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} +; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]} +; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]} ;. From d245be6638e80050dd3381f0aac8586621f4b59a Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 13 Oct 2025 11:27:33 +0100 Subject: [PATCH 43/60] [SCEV] Add test with ptrtoint guards and their order swapped. Additional test coverage for https://github.com/llvm/llvm-project/pull/160500. (cherry picked from commit 735ee5cc1f0d3566b548432d24a6b539a6a02af9) --- .../test/Analysis/ScalarEvolution/ptrtoint.ll | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll index e784d25385980..acac2c98ad174 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll @@ -447,6 +447,84 @@ bb5: ret void } +define void @pr46786_c26_char_cmp_ops_swapped(ptr %arg, ptr %arg1, ptr %arg2) { +; X64-LABEL: 'pr46786_c26_char_cmp_ops_swapped' +; X64-NEXT: Classifying expressions for: @pr46786_c26_char_cmp_ops_swapped +; X64-NEXT: %i4 = ptrtoint ptr %arg to i64 +; X64-NEXT: --> (ptrtoint ptr %arg to i64) U: full-set S: full-set +; X64-NEXT: %i7 = phi ptr [ %arg, %bb3 ], [ %i14, %bb6 ] +; X64-NEXT: --> {%arg,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg) LoopDispositions: { %bb6: Computable } +; X64-NEXT: %i8 = load i8, ptr %i7, align 1 +; X64-NEXT: --> %i8 U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } +; X64-NEXT: %i9 = ptrtoint ptr %i7 to i64 +; X64-NEXT: --> {(ptrtoint ptr %arg to i64),+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } +; X64-NEXT: %i10 = sub i64 %i9, %i4 +; X64-NEXT: --> {0,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } +; X64-NEXT: %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 +; X64-NEXT: --> {%arg2,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg2) LoopDispositions: { %bb6: Computable } +; X64-NEXT: %i12 = load i8, ptr %i11, align 1 +; X64-NEXT: --> %i12 U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } +; X64-NEXT: %i13 = add i8 %i12, %i8 +; X64-NEXT: --> (%i12 + %i8) U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } +; X64-NEXT: %i14 = getelementptr inbounds i8, ptr %i7, i64 1 +; X64-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg) LoopDispositions: { %bb6: Computable } +; X64-NEXT: Determining loop execution counts for: @pr46786_c26_char_cmp_ops_swapped +; X64-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) +; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -1 +; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) +; X64-NEXT: Loop %bb6: Trip multiple is 1 +; +; X32-LABEL: 'pr46786_c26_char_cmp_ops_swapped' +; X32-NEXT: Classifying expressions for: @pr46786_c26_char_cmp_ops_swapped +; X32-NEXT: %i4 = ptrtoint ptr %arg to i64 +; X32-NEXT: --> (zext i32 (ptrtoint ptr %arg to i32) to i64) U: [0,4294967296) S: [0,4294967296) +; X32-NEXT: %i7 = phi ptr [ %arg, %bb3 ], [ %i14, %bb6 ] +; X32-NEXT: --> {%arg,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg) LoopDispositions: { %bb6: Computable } +; X32-NEXT: %i8 = load i8, ptr %i7, align 1 +; X32-NEXT: --> %i8 U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } +; X32-NEXT: %i9 = ptrtoint ptr %i7 to i64 +; X32-NEXT: --> {(zext i32 (ptrtoint ptr %arg to i32) to i64),+,1}<%bb6> U: [0,8589934591) S: [0,8589934591) Exits: ((zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) + (zext i32 (ptrtoint ptr %arg to i32) to i64)) LoopDispositions: { %bb6: Computable } +; X32-NEXT: %i10 = sub i64 %i9, %i4 +; X32-NEXT: --> {0,+,1}<%bb6> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) LoopDispositions: { %bb6: Computable } +; X32-NEXT: %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 +; X32-NEXT: --> {%arg2,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg2) LoopDispositions: { %bb6: Computable } +; X32-NEXT: %i12 = load i8, ptr %i11, align 1 +; X32-NEXT: --> %i12 U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } +; X32-NEXT: %i13 = add i8 %i12, %i8 +; X32-NEXT: --> (%i12 + %i8) U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } +; X32-NEXT: %i14 = getelementptr inbounds i8, ptr %i7, i64 1 +; X32-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg) LoopDispositions: { %bb6: Computable } +; X32-NEXT: Determining loop execution counts for: @pr46786_c26_char_cmp_ops_swapped +; X32-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) +; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -1 +; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) +; X32-NEXT: Loop %bb6: Trip multiple is 1 +; + %i = icmp eq ptr %arg1, %arg + br i1 %i, label %bb5, label %bb3 + +bb3: + %i4 = ptrtoint ptr %arg to i64 + br label %bb6 + +bb6: + %i7 = phi ptr [ %arg, %bb3 ], [ %i14, %bb6 ] + %i8 = load i8, ptr %i7 + %i9 = ptrtoint ptr %i7 to i64 + %i10 = sub i64 %i9, %i4 + %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 + %i12 = load i8, ptr %i11 + %i13 = add i8 %i12, %i8 + store i8 %i13, ptr %i11 + %i14 = getelementptr inbounds i8, ptr %i7, i64 1 + %i15 = icmp eq ptr %i14, %arg1 + br i1 %i15, label %bb5, label %bb6 + +bb5: + ret void +} + + ; void pr46786_c26_int(int* start, int *end, int *other) { ; for (int* cur = start; cur != end; ++cur) ; other[cur - start] += *cur; From 198f96b96560caf94f3558d9cc17408efea2fed6 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 13 Oct 2025 18:12:31 +0100 Subject: [PATCH 44/60] [SCEV] Add m_scev_Trunc pattern matcher. (#163169) This patch adds a new m_scev_Trunc pattern matcher for SCEVTruncateExpr and uses it in a few places to slightly simplify the code. PR: https://github.com/llvm/llvm-project/pull/163169 (cherry picked from commit bc4e14b6daedcd4b0dd1f6c02909c88944f972d9) --- .../Analysis/ScalarEvolutionPatternMatch.h | 6 +++ llvm/lib/Analysis/ScalarEvolution.cpp | 49 ++++++++----------- 2 files changed, 27 insertions(+), 28 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index 011d5994dc670..33571cbfdcb90 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -170,6 +170,12 @@ m_scev_PtrToInt(const Op0_t &Op0) { return SCEVUnaryExpr_match(Op0); } +template +inline SCEVUnaryExpr_match +m_scev_Trunc(const Op0_t &Op0) { + return m_scev_Unary(Op0); +} + /// Match a binary SCEV. template struct SCEVBinaryExpr_match { diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 481fef9bcb7e6..1fceb85637827 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -5412,20 +5412,15 @@ static Type *isSimpleCastedPHI(const SCEV *Op, const SCEVUnknown *SymbolicPHI, if (SourceBits != NewBits) return nullptr; - const SCEVSignExtendExpr *SExt = dyn_cast(Op); - const SCEVZeroExtendExpr *ZExt = dyn_cast(Op); - if (!SExt && !ZExt) - return nullptr; - const SCEVTruncateExpr *Trunc = - SExt ? dyn_cast(SExt->getOperand()) - : dyn_cast(ZExt->getOperand()); - if (!Trunc) - return nullptr; - const SCEV *X = Trunc->getOperand(); - if (X != SymbolicPHI) - return nullptr; - Signed = SExt != nullptr; - return Trunc->getType(); + if (match(Op, m_scev_SExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) { + Signed = true; + return cast(Op)->getOperand()->getType(); + } + if (match(Op, m_scev_ZExt(m_scev_Trunc(m_scev_Specific(SymbolicPHI))))) { + Signed = false; + return cast(Op)->getOperand()->getType(); + } + return nullptr; } static const Loop *isIntegerLoopHeaderPHI(const PHINode *PN, LoopInfo &LI) { @@ -15371,20 +15366,18 @@ bool ScalarEvolution::matchURem(const SCEV *Expr, const SCEV *&LHS, // Try to match 'zext (trunc A to iB) to iY', which is used // for URem with constant power-of-2 second operands. Make sure the size of // the operand A matches the size of the whole expressions. - if (const auto *ZExt = dyn_cast(Expr)) - if (const auto *Trunc = dyn_cast(ZExt->getOperand(0))) { - LHS = Trunc->getOperand(); - // Bail out if the type of the LHS is larger than the type of the - // expression for now. - if (getTypeSizeInBits(LHS->getType()) > - getTypeSizeInBits(Expr->getType())) - return false; - if (LHS->getType() != Expr->getType()) - LHS = getZeroExtendExpr(LHS, Expr->getType()); - RHS = getConstant(APInt(getTypeSizeInBits(Expr->getType()), 1) - << getTypeSizeInBits(Trunc->getType())); - return true; - } + if (match(Expr, m_scev_ZExt(m_scev_Trunc(m_SCEV(LHS))))) { + Type *TruncTy = cast(Expr)->getOperand()->getType(); + // Bail out if the type of the LHS is larger than the type of the + // expression for now. + if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(Expr->getType())) + return false; + if (LHS->getType() != Expr->getType()) + LHS = getZeroExtendExpr(LHS, Expr->getType()); + RHS = getConstant(APInt(getTypeSizeInBits(Expr->getType()), 1) + << getTypeSizeInBits(TruncTy)); + return true; + } const auto *Add = dyn_cast(Expr); if (Add == nullptr || Add->getNumOperands() != 2) return false; From 2af416bfcdf0e78c140ae7ca1d9ef5c1cda50860 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 13 Oct 2025 20:28:53 +0100 Subject: [PATCH 45/60] [SCEV] Move URem matching to ScalarEvolutionPatternMatch.h (#163170) Move URem matching to ScalarEvolutionPatternMatch.h so it can be re-used together with other matchers. Depends on https://github.com/llvm/llvm-project/pull/163169 PR: https://github.com/llvm/llvm-project/pull/163170 (cherry picked from commit 7f04ee19d21d28f7a533fff98c69c16863e6984a) --- llvm/include/llvm/Analysis/ScalarEvolution.h | 4 - .../Analysis/ScalarEvolutionPatternMatch.h | 74 +++++++++++++ llvm/lib/Analysis/ScalarEvolution.cpp | 102 ++++-------------- .../Utils/ScalarEvolutionExpander.cpp | 2 +- .../Analysis/ScalarEvolutionTest.cpp | 12 +-- 5 files changed, 98 insertions(+), 96 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index faebf07afc107..3b11fd86bdc99 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -2316,10 +2316,6 @@ class ScalarEvolution { /// an add rec on said loop. void getUsedLoops(const SCEV *S, SmallPtrSetImpl &LoopsUsed); - /// Try to match the pattern generated by getURemExpr(A, B). If successful, - /// Assign A and B to LHS and RHS, respectively. - LLVM_ABI bool matchURem(const SCEV *Expr, const SCEV *&LHS, const SCEV *&RHS); - /// Look for a SCEV expression with type `SCEVType` and operands `Ops` in /// `UniqueSCEVs`. Return if found, else nullptr. SCEV *findExistingSCEVInCache(SCEVTypes SCEVType, ArrayRef Ops); diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index 33571cbfdcb90..e30e94536993e 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -215,6 +215,80 @@ m_scev_UDiv(const Op0_t &Op0, const Op1_t &Op1) { return m_scev_Binary(Op0, Op1); } +/// Match unsigned remainder pattern. +/// Matches patterns generated by getURemExpr. +template struct SCEVURem_match { + Op0_t Op0; + Op1_t Op1; + ScalarEvolution &SE; + + SCEVURem_match(Op0_t Op0, Op1_t Op1, ScalarEvolution &SE) + : Op0(Op0), Op1(Op1), SE(SE) {} + + bool match(const SCEV *Expr) const { + if (Expr->getType()->isPointerTy()) + return false; + + // Try to match 'zext (trunc A to iB) to iY', which is used + // for URem with constant power-of-2 second operands. Make sure the size of + // the operand A matches the size of the whole expressions. + const SCEV *LHS; + if (SCEVPatternMatch::match(Expr, m_scev_ZExt(m_scev_Trunc(m_SCEV(LHS))))) { + Type *TruncTy = cast(Expr)->getOperand()->getType(); + // Bail out if the type of the LHS is larger than the type of the + // expression for now. + if (SE.getTypeSizeInBits(LHS->getType()) > + SE.getTypeSizeInBits(Expr->getType())) + return false; + if (LHS->getType() != Expr->getType()) + LHS = SE.getZeroExtendExpr(LHS, Expr->getType()); + const SCEV *RHS = + SE.getConstant(APInt(SE.getTypeSizeInBits(Expr->getType()), 1) + << SE.getTypeSizeInBits(TruncTy)); + return Op0.match(LHS) && Op1.match(RHS); + } + const auto *Add = dyn_cast(Expr); + if (Add == nullptr || Add->getNumOperands() != 2) + return false; + + const SCEV *A = Add->getOperand(1); + const auto *Mul = dyn_cast(Add->getOperand(0)); + + if (Mul == nullptr) + return false; + + const auto MatchURemWithDivisor = [&](const SCEV *B) { + // (SomeExpr + (-(SomeExpr / B) * B)). + if (Expr == SE.getURemExpr(A, B)) + return Op0.match(A) && Op1.match(B); + return false; + }; + + // (SomeExpr + (-1 * (SomeExpr / B) * B)). + if (Mul->getNumOperands() == 3 && isa(Mul->getOperand(0))) + return MatchURemWithDivisor(Mul->getOperand(1)) || + MatchURemWithDivisor(Mul->getOperand(2)); + + // (SomeExpr + ((-SomeExpr / B) * B)) or (SomeExpr + ((SomeExpr / B) * -B)). + if (Mul->getNumOperands() == 2) + return MatchURemWithDivisor(Mul->getOperand(1)) || + MatchURemWithDivisor(Mul->getOperand(0)) || + MatchURemWithDivisor(SE.getNegativeSCEV(Mul->getOperand(1))) || + MatchURemWithDivisor(SE.getNegativeSCEV(Mul->getOperand(0))); + return false; + } +}; + +/// Match the mathematical pattern A - (A / B) * B, where A and B can be +/// arbitrary expressions. Also match zext (trunc A to iB) to iY, which is used +/// for URem with constant power-of-2 second operands. It's not always easy, as +/// A and B can be folded (imagine A is X / 2, and B is 4, A / B becomes X / 8). +template +inline SCEVURem_match m_scev_URem(Op0_t LHS, Op1_t RHS, + ScalarEvolution &SE) { + return SCEVURem_match(LHS, RHS, SE); +} + inline class_match m_Loop() { return class_match(); } /// Match an affine SCEVAddRecExpr. diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 1fceb85637827..279246816e101 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -1774,7 +1774,7 @@ const SCEV *ScalarEvolution::getZeroExtendExprImpl(const SCEV *Op, Type *Ty, { const SCEV *LHS; const SCEV *RHS; - if (matchURem(Op, LHS, RHS)) + if (match(Op, m_scev_URem(m_SCEV(LHS), m_SCEV(RHS), *this))) return getURemExpr(getZeroExtendExpr(LHS, Ty, Depth + 1), getZeroExtendExpr(RHS, Ty, Depth + 1)); } @@ -2699,17 +2699,12 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, } // Canonicalize (-1 * urem X, Y) + X --> (Y * X/Y) - if (Ops.size() == 2) { - const SCEVMulExpr *Mul = dyn_cast(Ops[0]); - if (Mul && Mul->getNumOperands() == 2 && - Mul->getOperand(0)->isAllOnesValue()) { - const SCEV *X; - const SCEV *Y; - if (matchURem(Mul->getOperand(1), X, Y) && X == Ops[1]) { - return getMulExpr(Y, getUDivExpr(X, Y)); - } - } - } + const SCEV *Y; + if (Ops.size() == 2 && + match(Ops[0], + m_scev_Mul(m_scev_AllOnes(), + m_scev_URem(m_scev_Specific(Ops[1]), m_SCEV(Y), *this)))) + return getMulExpr(Y, getUDivExpr(Ops[1], Y)); // Skip past any other cast SCEVs. while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scAddExpr) @@ -15353,65 +15348,6 @@ void PredicatedScalarEvolution::print(raw_ostream &OS, unsigned Depth) const { } } -// Match the mathematical pattern A - (A / B) * B, where A and B can be -// arbitrary expressions. Also match zext (trunc A to iB) to iY, which is used -// for URem with constant power-of-2 second operands. -// It's not always easy, as A and B can be folded (imagine A is X / 2, and B is -// 4, A / B becomes X / 8). -bool ScalarEvolution::matchURem(const SCEV *Expr, const SCEV *&LHS, - const SCEV *&RHS) { - if (Expr->getType()->isPointerTy()) - return false; - - // Try to match 'zext (trunc A to iB) to iY', which is used - // for URem with constant power-of-2 second operands. Make sure the size of - // the operand A matches the size of the whole expressions. - if (match(Expr, m_scev_ZExt(m_scev_Trunc(m_SCEV(LHS))))) { - Type *TruncTy = cast(Expr)->getOperand()->getType(); - // Bail out if the type of the LHS is larger than the type of the - // expression for now. - if (getTypeSizeInBits(LHS->getType()) > getTypeSizeInBits(Expr->getType())) - return false; - if (LHS->getType() != Expr->getType()) - LHS = getZeroExtendExpr(LHS, Expr->getType()); - RHS = getConstant(APInt(getTypeSizeInBits(Expr->getType()), 1) - << getTypeSizeInBits(TruncTy)); - return true; - } - const auto *Add = dyn_cast(Expr); - if (Add == nullptr || Add->getNumOperands() != 2) - return false; - - const SCEV *A = Add->getOperand(1); - const auto *Mul = dyn_cast(Add->getOperand(0)); - - if (Mul == nullptr) - return false; - - const auto MatchURemWithDivisor = [&](const SCEV *B) { - // (SomeExpr + (-(SomeExpr / B) * B)). - if (Expr == getURemExpr(A, B)) { - LHS = A; - RHS = B; - return true; - } - return false; - }; - - // (SomeExpr + (-1 * (SomeExpr / B) * B)). - if (Mul->getNumOperands() == 3 && isa(Mul->getOperand(0))) - return MatchURemWithDivisor(Mul->getOperand(1)) || - MatchURemWithDivisor(Mul->getOperand(2)); - - // (SomeExpr + ((-SomeExpr / B) * B)) or (SomeExpr + ((SomeExpr / B) * -B)). - if (Mul->getNumOperands() == 2) - return MatchURemWithDivisor(Mul->getOperand(1)) || - MatchURemWithDivisor(Mul->getOperand(0)) || - MatchURemWithDivisor(getNegativeSCEV(Mul->getOperand(1))) || - MatchURemWithDivisor(getNegativeSCEV(Mul->getOperand(0))); - return false; -} - ScalarEvolution::LoopGuards ScalarEvolution::LoopGuards::collect(const Loop *L, ScalarEvolution &SE) { BasicBlock *Header = L->getHeader(); @@ -15623,20 +15559,18 @@ void ScalarEvolution::LoopGuards::collectFromBlock( if (Predicate == CmpInst::ICMP_EQ && match(RHS, m_scev_Zero())) { // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to // explicitly express that. - const SCEV *URemLHS = nullptr; + const SCEVUnknown *URemLHS = nullptr; const SCEV *URemRHS = nullptr; - if (SE.matchURem(LHS, URemLHS, URemRHS)) { - if (const SCEVUnknown *LHSUnknown = dyn_cast(URemLHS)) { - auto I = RewriteMap.find(LHSUnknown); - const SCEV *RewrittenLHS = - I != RewriteMap.end() ? I->second : LHSUnknown; - RewrittenLHS = ApplyDivisibiltyOnMinMaxExpr(RewrittenLHS, URemRHS); - const auto *Multiple = - SE.getMulExpr(SE.getUDivExpr(RewrittenLHS, URemRHS), URemRHS); - RewriteMap[LHSUnknown] = Multiple; - ExprsToRewrite.push_back(LHSUnknown); - return; - } + if (match(LHS, + m_scev_URem(m_SCEVUnknown(URemLHS), m_SCEV(URemRHS), SE))) { + auto I = RewriteMap.find(URemLHS); + const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : URemLHS; + RewrittenLHS = ApplyDivisibiltyOnMinMaxExpr(RewrittenLHS, URemRHS); + const auto *Multiple = + SE.getMulExpr(SE.getUDivExpr(RewrittenLHS, URemRHS), URemRHS); + RewriteMap[URemLHS] = Multiple; + ExprsToRewrite.push_back(URemLHS); + return; } } diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index e0b3a5b7892a1..aff0b901bd031 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -506,7 +506,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) { // Recognize the canonical representation of an unsimplifed urem. const SCEV *URemLHS = nullptr; const SCEV *URemRHS = nullptr; - if (SE.matchURem(S, URemLHS, URemRHS)) { + if (match(S, m_scev_URem(m_SCEV(URemLHS), m_SCEV(URemRHS), SE))) { Value *LHS = expand(URemLHS); Value *RHS = expand(URemRHS); return InsertBinop(Instruction::URem, LHS, RHS, SCEV::FlagAnyWrap, diff --git a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp index 678960418d7d7..d2b506af8d398 100644 --- a/llvm/unittests/Analysis/ScalarEvolutionTest.cpp +++ b/llvm/unittests/Analysis/ScalarEvolutionTest.cpp @@ -11,6 +11,7 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ScalarEvolutionNormalization.h" +#include "llvm/Analysis/ScalarEvolutionPatternMatch.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/Constants.h" @@ -26,6 +27,8 @@ namespace llvm { +using namespace SCEVPatternMatch; + // We use this fixture to ensure that we clean up ScalarEvolution before // deleting the PassManager. class ScalarEvolutionsTest : public testing::Test { @@ -64,11 +67,6 @@ static std::optional computeConstantDifference(ScalarEvolution &SE, return SE.computeConstantDifference(LHS, RHS); } - static bool matchURem(ScalarEvolution &SE, const SCEV *Expr, const SCEV *&LHS, - const SCEV *&RHS) { - return SE.matchURem(Expr, LHS, RHS); - } - static bool isImpliedCond( ScalarEvolution &SE, ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS, ICmpInst::Predicate FoundPred, const SCEV *FoundLHS, @@ -1524,7 +1522,7 @@ TEST_F(ScalarEvolutionsTest, MatchURem) { auto *URemI = getInstructionByName(F, N); auto *S = SE.getSCEV(URemI); const SCEV *LHS, *RHS; - EXPECT_TRUE(matchURem(SE, S, LHS, RHS)); + EXPECT_TRUE(match(S, m_scev_URem(m_SCEV(LHS), m_SCEV(RHS), SE))); EXPECT_EQ(LHS, SE.getSCEV(URemI->getOperand(0))); EXPECT_EQ(RHS, SE.getSCEV(URemI->getOperand(1))); EXPECT_EQ(LHS->getType(), S->getType()); @@ -1537,7 +1535,7 @@ TEST_F(ScalarEvolutionsTest, MatchURem) { auto *URem1 = getInstructionByName(F, "rem4"); auto *S = SE.getSCEV(Ext); const SCEV *LHS, *RHS; - EXPECT_TRUE(matchURem(SE, S, LHS, RHS)); + EXPECT_TRUE(match(S, m_scev_URem(m_SCEV(LHS), m_SCEV(RHS), SE))); EXPECT_NE(LHS, SE.getSCEV(URem1->getOperand(0))); // RHS and URem1->getOperand(1) have different widths, so compare the // integer values. From d3f487a5461eb281d70bc0e4a414f9e856da77b4 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 14 Oct 2025 15:20:34 +0100 Subject: [PATCH 46/60] [SCEV] Collect guard info for ICMP NE w/o constants. (#160500) When collecting information from loop guards, use UMax(1, %b - %a) for ICMP NE %a, %b, if neither are constant. This improves results in some cases, and will be even more useful together with * https://github.com/llvm/llvm-project/pull/160012 * https://github.com/llvm/llvm-project/pull/159942 https://alive2.llvm.org/ce/z/YyBvoT PR: https://github.com/llvm/llvm-project/pull/160500 (cherry picked from commit 2d027260b0f8ef9b2e0b2fc8c254b2b8da0ae4f7) --- llvm/lib/Analysis/ScalarEvolution.cpp | 15 +++++ .../test/Analysis/ScalarEvolution/ptrtoint.ll | 20 +++---- .../IndVarSimplify/pointer-loop-guards.ll | 50 ++++++++++++++++ llvm/test/Transforms/LoopUnroll/scevunroll.ll | 60 +++++++++++++++++++ 4 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 279246816e101..4074282b99995 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15695,6 +15695,21 @@ void ScalarEvolution::LoopGuards::collectFromBlock( const SCEV *OneAlignedUp = GetNextSCEVDividesByDivisor(One, DividesBy); To = SE.getUMaxExpr(FromRewritten, OneAlignedUp); + } else { + if (LHS->getType()->isPointerTy()) { + LHS = SE.getLosslessPtrToIntExpr(LHS); + RHS = SE.getLosslessPtrToIntExpr(RHS); + if (isa(LHS) || isa(RHS)) + break; + } + auto AddSubRewrite = [&](const SCEV *A, const SCEV *B) { + const SCEV *Sub = SE.getMinusSCEV(A, B); + AddRewrite(Sub, Sub, + SE.getUMaxExpr(Sub, SE.getOne(From->getType()))); + }; + AddSubRewrite(LHS, RHS); + AddSubRewrite(RHS, LHS); + continue; } break; default: diff --git a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll index acac2c98ad174..0c1f37bf58601 100644 --- a/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll +++ b/llvm/test/Analysis/ScalarEvolution/ptrtoint.ll @@ -382,7 +382,7 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: %i9 = ptrtoint ptr %i7 to i64 ; X64-NEXT: --> {(ptrtoint ptr %arg to i64),+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } ; X64-NEXT: %i10 = sub i64 %i9, %i4 -; X64-NEXT: --> {0,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } +; X64-NEXT: --> {0,+,1}<%bb6> U: [0,-1) S: [0,-1) Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } ; X64-NEXT: %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 ; X64-NEXT: --> {%arg2,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg2) LoopDispositions: { %bb6: Computable } ; X64-NEXT: %i12 = load i8, ptr %i11, align 1 @@ -393,7 +393,7 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg) LoopDispositions: { %bb6: Computable } ; X64-NEXT: Determining loop execution counts for: @pr46786_c26_char ; X64-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) -; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -1 +; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -2 ; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) ; X64-NEXT: Loop %bb6: Trip multiple is 1 ; @@ -406,9 +406,9 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: %i8 = load i8, ptr %i7, align 1 ; X32-NEXT: --> %i8 U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } ; X32-NEXT: %i9 = ptrtoint ptr %i7 to i64 -; X32-NEXT: --> {(zext i32 (ptrtoint ptr %arg to i32) to i64),+,1}<%bb6> U: [0,8589934591) S: [0,8589934591) Exits: ((zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) + (zext i32 (ptrtoint ptr %arg to i32) to i64)) LoopDispositions: { %bb6: Computable } +; X32-NEXT: --> {(zext i32 (ptrtoint ptr %arg to i32) to i64),+,1}<%bb6> U: [0,8589934590) S: [0,8589934590) Exits: ((zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) + (zext i32 (ptrtoint ptr %arg to i32) to i64)) LoopDispositions: { %bb6: Computable } ; X32-NEXT: %i10 = sub i64 %i9, %i4 -; X32-NEXT: --> {0,+,1}<%bb6> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) LoopDispositions: { %bb6: Computable } +; X32-NEXT: --> {0,+,1}<%bb6> U: [0,4294967295) S: [0,4294967295) Exits: (zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) LoopDispositions: { %bb6: Computable } ; X32-NEXT: %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 ; X32-NEXT: --> {%arg2,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg2) LoopDispositions: { %bb6: Computable } ; X32-NEXT: %i12 = load i8, ptr %i11, align 1 @@ -419,7 +419,7 @@ define void @pr46786_c26_char(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg) LoopDispositions: { %bb6: Computable } ; X32-NEXT: Determining loop execution counts for: @pr46786_c26_char ; X32-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) -; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -1 +; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -2 ; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) ; X32-NEXT: Loop %bb6: Trip multiple is 1 ; @@ -459,7 +459,7 @@ define void @pr46786_c26_char_cmp_ops_swapped(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: %i9 = ptrtoint ptr %i7 to i64 ; X64-NEXT: --> {(ptrtoint ptr %arg to i64),+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } ; X64-NEXT: %i10 = sub i64 %i9, %i4 -; X64-NEXT: --> {0,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } +; X64-NEXT: --> {0,+,1}<%bb6> U: [0,-1) S: [0,-1) Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) LoopDispositions: { %bb6: Computable } ; X64-NEXT: %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 ; X64-NEXT: --> {%arg2,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg2) LoopDispositions: { %bb6: Computable } ; X64-NEXT: %i12 = load i8, ptr %i11, align 1 @@ -470,7 +470,7 @@ define void @pr46786_c26_char_cmp_ops_swapped(ptr %arg, ptr %arg1, ptr %arg2) { ; X64-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64) + %arg) LoopDispositions: { %bb6: Computable } ; X64-NEXT: Determining loop execution counts for: @pr46786_c26_char_cmp_ops_swapped ; X64-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) -; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -1 +; X64-NEXT: Loop %bb6: constant max backedge-taken count is i64 -2 ; X64-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i64)) + (ptrtoint ptr %arg1 to i64)) ; X64-NEXT: Loop %bb6: Trip multiple is 1 ; @@ -483,9 +483,9 @@ define void @pr46786_c26_char_cmp_ops_swapped(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: %i8 = load i8, ptr %i7, align 1 ; X32-NEXT: --> %i8 U: full-set S: full-set Exits: <> LoopDispositions: { %bb6: Variant } ; X32-NEXT: %i9 = ptrtoint ptr %i7 to i64 -; X32-NEXT: --> {(zext i32 (ptrtoint ptr %arg to i32) to i64),+,1}<%bb6> U: [0,8589934591) S: [0,8589934591) Exits: ((zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) + (zext i32 (ptrtoint ptr %arg to i32) to i64)) LoopDispositions: { %bb6: Computable } +; X32-NEXT: --> {(zext i32 (ptrtoint ptr %arg to i32) to i64),+,1}<%bb6> U: [0,8589934590) S: [0,8589934590) Exits: ((zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) + (zext i32 (ptrtoint ptr %arg to i32) to i64)) LoopDispositions: { %bb6: Computable } ; X32-NEXT: %i10 = sub i64 %i9, %i4 -; X32-NEXT: --> {0,+,1}<%bb6> U: [0,4294967296) S: [0,4294967296) Exits: (zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) LoopDispositions: { %bb6: Computable } +; X32-NEXT: --> {0,+,1}<%bb6> U: [0,4294967295) S: [0,4294967295) Exits: (zext i32 (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) to i64) LoopDispositions: { %bb6: Computable } ; X32-NEXT: %i11 = getelementptr inbounds i8, ptr %arg2, i64 %i10 ; X32-NEXT: --> {%arg2,+,1}<%bb6> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg2) LoopDispositions: { %bb6: Computable } ; X32-NEXT: %i12 = load i8, ptr %i11, align 1 @@ -496,7 +496,7 @@ define void @pr46786_c26_char_cmp_ops_swapped(ptr %arg, ptr %arg1, ptr %arg2) { ; X32-NEXT: --> {(1 + %arg),+,1}<%bb6> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32) + %arg) LoopDispositions: { %bb6: Computable } ; X32-NEXT: Determining loop execution counts for: @pr46786_c26_char_cmp_ops_swapped ; X32-NEXT: Loop %bb6: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) -; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -1 +; X32-NEXT: Loop %bb6: constant max backedge-taken count is i32 -2 ; X32-NEXT: Loop %bb6: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %arg to i32)) + (ptrtoint ptr %arg1 to i32)) ; X32-NEXT: Loop %bb6: Trip multiple is 1 ; diff --git a/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll b/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll new file mode 100644 index 0000000000000..9371fe24be05d --- /dev/null +++ b/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll @@ -0,0 +1,50 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -p indvars -S %s | FileCheck %s + +declare i1 @cond() + +define i64 @test_ptr_compare_guard(ptr %start, ptr %end) { +; CHECK-LABEL: define i64 @test_ptr_compare_guard( +; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[C_0:%.*]] = icmp eq ptr [[START]], [[END]] +; CHECK-NEXT: br i1 [[C_0]], label %[[EXIT:.*]], label %[[LOOP_HEADER_PREHEADER:.*]] +; CHECK: [[LOOP_HEADER_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[PTR_IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[START]], %[[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[I64_IV:%.*]] = phi i64 [ [[I64_IV_NEXT:%.*]], %[[LOOP_LATCH]] ], [ 0, %[[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[C_1:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[EXIT_LOOPEXIT:.*]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr i8, ptr [[PTR_IV]], i64 1 +; CHECK-NEXT: [[I64_IV_NEXT]] = add nuw i64 [[I64_IV]], 1 +; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] +; CHECK-NEXT: br i1 [[C_2]], label %[[EXIT_LOOPEXIT]], label %[[LOOP_HEADER]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: [[RES_PH:%.*]] = phi i64 [ 0, %[[LOOP_HEADER]] ], [ [[I64_IV]], %[[LOOP_LATCH]] ] +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[RES:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[RES_PH]], %[[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret i64 [[RES]] +; +entry: + %c.0 = icmp eq ptr %start, %end + br i1 %c.0, label %exit, label %loop.header + +loop.header: + %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop.latch ] + %i64.iv = phi i64 [ 0, %entry ], [ %i64.iv.next, %loop.latch ] + %c.1 = call i1 @cond() + br i1 %c.1, label %loop.latch, label %exit + +loop.latch: + %ptr.iv.next = getelementptr i8, ptr %ptr.iv, i64 1 + %i64.iv.next = add i64 %i64.iv, 1 + %c.2 = icmp eq ptr %ptr.iv.next, %end + br i1 %c.2, label %exit, label %loop.header + +exit: + %res = phi i64 [ 0, %entry ], [ %i64.iv, %loop.latch ], [ 0, %loop.header ] + ret i64 %res +} diff --git a/llvm/test/Transforms/LoopUnroll/scevunroll.ll b/llvm/test/Transforms/LoopUnroll/scevunroll.ll index b6b14e365cc1d..a3cd07248c54b 100644 --- a/llvm/test/Transforms/LoopUnroll/scevunroll.ll +++ b/llvm/test/Transforms/LoopUnroll/scevunroll.ll @@ -435,3 +435,63 @@ exit: } declare void @fn(i32) + +define void @peel_int_eq_condition(i32 %start) { +; CHECK-LABEL: @peel_int_eq_condition( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[START:%.*]], i32 100) +; CHECK-NEXT: [[TMP0:%.*]] = add nuw i32 [[SMAX]], 1 +; CHECK-NEXT: br label [[LOOP_PEEL_BEGIN:%.*]] +; CHECK: loop.peel.begin: +; CHECK-NEXT: br label [[LOOP_PEEL:%.*]] +; CHECK: loop.peel: +; CHECK-NEXT: [[C_0_PEEL:%.*]] = icmp eq i32 [[START]], [[START]] +; CHECK-NEXT: br i1 [[C_0_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[LOOP_LATCH_PEEL:%.*]] +; CHECK: if.then.peel: +; CHECK-NEXT: call void @fn(i32 [[START]]) +; CHECK-NEXT: br label [[LOOP_LATCH_PEEL]] +; CHECK: loop.latch.peel: +; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i32 [[START]], 1 +; CHECK-NEXT: [[EXITCOND_PEEL:%.*]] = icmp ne i32 [[IV_NEXT_PEEL]], [[TMP0]] +; CHECK-NEXT: br i1 [[EXITCOND_PEEL]], label [[LOOP_PEEL_NEXT:%.*]], label [[EXIT:%.*]] +; CHECK: loop.peel.next: +; CHECK-NEXT: br label [[LOOP_PEEL_NEXT1:%.*]] +; CHECK: loop.peel.next1: +; CHECK-NEXT: br label [[ENTRY_PEEL_NEWPH:%.*]] +; CHECK: entry.peel.newph: +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT_PEEL]], [[ENTRY_PEEL_NEWPH]] ], [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ] +; CHECK-NEXT: br i1 false, label [[IF_THEN:%.*]], label [[LOOP_LATCH]] +; CHECK: if.then: +; CHECK-NEXT: call void @fn(i32 [[IV]]) +; CHECK-NEXT: br label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]] +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK: exit.loopexit: +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i32 [ %start, %entry ], [ %iv.next, %loop.latch ] + %c.0 = icmp eq i32 %iv, %start + br i1 %c.0, label %if.then, label %loop.latch + +if.then: + call void @fn(i32 %iv) + br label %loop.latch + +loop.latch: + %iv.next = add i32 %iv, 1 + %ec = icmp slt i32 %iv, 100 + br i1 %ec, label %loop, label %exit + +exit: + ret void +} From 354788fe093d9a02fa6ef0316cb6dec6ff346133 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 14 Oct 2025 20:18:39 +0100 Subject: [PATCH 47/60] [SCEV] Use context instruction for SCEVUnknowns in getConstantMultiple. (#163260) Follow-up to https://github.com/llvm/llvm-project/pull/160941. Even if we don't have a context instruction for the caller, we should be able to provide context instructions for SCEVUnknowns. Unless I am missing something, SCEVUnknown only become available at the point their underlying IR instruction has been defined. If it is an argument, it should be safe to use the first instruction in the entry block or the instruction itself if it wraps an instruction. This allows getConstantMultiple to make better use of alignment assumptions. PR: https://github.com/llvm/llvm-project/pull/163260 (cherry picked from commit 3b46556cb7034409a4bb2566c6d0df7620c56bba) --- llvm/lib/Analysis/ScalarEvolution.cpp | 12 +++++++- .../trip-multiple-guard-info.ll | 28 +++++++++---------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 4074282b99995..1ee8a56d66f55 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -6410,8 +6410,18 @@ APInt ScalarEvolution::getConstantMultipleImpl(const SCEV *S, case scSequentialUMinExpr: return GetGCDMultiple(cast(S)); case scUnknown: { - // ask ValueTracking for known bits + // Ask ValueTracking for known bits. SCEVUnknown only become available at + // the point their underlying IR instruction has been defined. If CtxI was + // not provided, use: + // * the first instruction in the entry block if it is an argument + // * the instruction itself otherwise. const SCEVUnknown *U = cast(S); + if (!CtxI) { + if (isa(U->getValue())) + CtxI = &*F.getEntryBlock().begin(); + else if (auto *I = dyn_cast(U->getValue())) + CtxI = I; + } unsigned Known = computeKnownBits(U->getValue(), getDataLayout(), &AC, CtxI, &DT) .countMinTrailingZeros(); diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll index 7ba422da79ad8..a477465cb0699 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll @@ -578,22 +578,22 @@ define void @test_ptr_aligned_by_2_and_4_via_assumption(ptr %start, ptr %end) { ; CHECK-LABEL: 'test_ptr_aligned_by_2_and_4_via_assumption' ; CHECK-NEXT: Classifying expressions for: @test_ptr_aligned_by_2_and_4_via_assumption ; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,4}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 -; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_ptr_aligned_by_2_and_4_via_assumption ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Equal predicate: (zext i2 (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2)) to i64) == 0 ; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Equal predicate: (zext i2 (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2)) to i64) == 0 ; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Equal predicate: (zext i2 (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2)) to i64) == 0 ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 2) ] @@ -615,9 +615,9 @@ define void @test_ptrs_aligned_by_4_via_assumption(ptr %start, ptr %end) { ; CHECK-LABEL: 'test_ptrs_aligned_by_4_via_assumption' ; CHECK-NEXT: Classifying expressions for: @test_ptrs_aligned_by_4_via_assumption ; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,4}<%loop> U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: (-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 -; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: ((-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_ptrs_aligned_by_4_via_assumption ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 @@ -644,9 +644,9 @@ define void @test_ptrs_aligned_by_8_via_assumption(ptr %start, ptr %end) { ; CHECK-LABEL: 'test_ptrs_aligned_by_8_via_assumption' ; CHECK-NEXT: Classifying expressions for: @test_ptrs_aligned_by_8_via_assumption ; CHECK-NEXT: %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: ((4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,4}<%loop> U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: (-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 -; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: (4 + (4 * ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4)) + %start) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: [0,-3) S: [-9223372036854775808,9223372036854775805) Exits: ((-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64) + %start) LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_ptrs_aligned_by_8_via_assumption ; CHECK-NEXT: Loop %loop: backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 4611686018427387903 @@ -677,22 +677,22 @@ define void @test_ptr_aligned_by_4_via_assumption_multiple_loop_predecessors(ptr ; CHECK-NEXT: %c = call i1 @cond() ; CHECK-NEXT: --> %c U: full-set S: full-set ; CHECK-NEXT: %iv = phi ptr [ %start, %then ], [ %start, %else ], [ %iv.next, %loop ] -; CHECK-NEXT: --> {%start,+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {%start,+,4}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 4 -; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: --> {(4 + %start),+,4}<%loop> U: [0,-1) S: [-9223372036854775808,9223372036854775807) Exits: <> LoopDispositions: { %loop: Computable } ; CHECK-NEXT: Determining loop execution counts for: @test_ptr_aligned_by_4_via_assumption_multiple_loop_predecessors ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. ; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. ; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Equal predicate: (zext i2 (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2)) to i64) == 0 ; CHECK-NEXT: Loop %loop: Predicated constant max backedge-taken count is i64 4611686018427387903 ; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Equal predicate: (zext i2 (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2)) to i64) == 0 ; CHECK-NEXT: Loop %loop: Predicated symbolic max backedge-taken count is ((-4 + (-1 * (ptrtoint ptr %start to i64)) + (ptrtoint ptr %end to i64)) /u 4) ; CHECK-NEXT: Predicates: -; CHECK-NEXT: Equal predicate: (zext i2 ((trunc i64 (ptrtoint ptr %end to i64) to i2) + (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2))) to i64) == 0 +; CHECK-NEXT: Equal predicate: (zext i2 (-1 * (trunc i64 (ptrtoint ptr %start to i64) to i2)) to i64) == 0 ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %start, i64 2) ] From 711a0600c5edf870c70d091a91597109695668d5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 16 Oct 2025 13:04:04 +0100 Subject: [PATCH 48/60] [SCEV] Add tests with multiple NE guards and different orders. Add additional test coverage for using NE guards added in 2d027260b (https://github.com/llvm/llvm-project/pull/160500) (cherry picked from commit 0792478e4e133be96650444f3264e89d002fc058) --- .../ne-guard-multiple-trip-count.ll | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 llvm/test/Analysis/ScalarEvolution/ne-guard-multiple-trip-count.ll diff --git a/llvm/test/Analysis/ScalarEvolution/ne-guard-multiple-trip-count.ll b/llvm/test/Analysis/ScalarEvolution/ne-guard-multiple-trip-count.ll new file mode 100644 index 0000000000000..220c5a1deb1a0 --- /dev/null +++ b/llvm/test/Analysis/ScalarEvolution/ne-guard-multiple-trip-count.ll @@ -0,0 +1,72 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -passes='print' -disable-output %s 2>&1 | FileCheck %s + +declare void @foo() + +; Tests with multiple guards for the same value and different values. + +define void @test_guard_order_b_then_c_and_d(ptr %a, ptr %b, ptr %c, ptr %d) { +; CHECK-LABEL: 'test_guard_order_b_then_c_and_d' +; CHECK-NEXT: Classifying expressions for: @test_guard_order_b_then_c_and_d +; CHECK-NEXT: %iv = phi ptr [ %a, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {%a,+,1}<%loop> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64) + %a) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 1 +; CHECK-NEXT: --> {(1 + %a),+,1}<%loop> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64) + %a) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_guard_order_b_then_c_and_d +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -2 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %cmp.eq.b = icmp ne ptr %a, %b + %cmp.eq.c = icmp ne ptr %a, %c + %cmp.eq.d = icmp ne ptr %b, %d + call void @llvm.assume(i1 %cmp.eq.b) + call void @llvm.assume(i1 %cmp.eq.c) + call void @llvm.assume(i1 %cmp.eq.d) + br label %loop + +loop: + %iv = phi ptr [ %a, %entry ], [ %iv.next, %loop ] + %iv.next = getelementptr i8, ptr %iv, i64 1 + call void @foo() + %ec = icmp eq ptr %iv.next, %b + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @test_guard_order_d_then_c_and_b(ptr %a, ptr %b, ptr %c, ptr %d) { +; CHECK-LABEL: 'test_guard_order_d_then_c_and_b' +; CHECK-NEXT: Classifying expressions for: @test_guard_order_d_then_c_and_b +; CHECK-NEXT: %iv = phi ptr [ %a, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {%a,+,1}<%loop> U: full-set S: full-set Exits: (-1 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64) + %a) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %iv.next = getelementptr i8, ptr %iv, i64 1 +; CHECK-NEXT: --> {(1 + %a),+,1}<%loop> U: full-set S: full-set Exits: ((-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64) + %a) LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_guard_order_d_then_c_and_b +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 -2 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %cmp.eq.b = icmp ne ptr %a, %b + %cmp.eq.c = icmp ne ptr %a, %c + %cmp.eq.d = icmp ne ptr %b, %d + call void @llvm.assume(i1 %cmp.eq.d) + call void @llvm.assume(i1 %cmp.eq.c) + call void @llvm.assume(i1 %cmp.eq.b) + br label %loop + +loop: + %iv = phi ptr [ %a, %entry ], [ %iv.next, %loop ] + %iv.next = getelementptr i8, ptr %iv, i64 1 + call void @foo() + %ec = icmp eq ptr %iv.next, %b + br i1 %ec, label %exit, label %loop + +exit: + ret void +} From 96dbb37b9184a989a2ec2fd5974095b0ddc31aa8 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 16 Oct 2025 18:50:01 +0100 Subject: [PATCH 49/60] [SCEV] Use m_scev_Mul in a few more places. (NFC) (#163364) Add a new variant of m_scev_Mul that binds a SCEVMulExpr and use it in SCEVURem_match and also update 2 more places in ScalarEvolution.cpp that can use m_scev_Mul as well. PR: https://github.com/llvm/llvm-project/pull/163364 (cherry picked from commit 7c54c8245a7164abb1f6931f3f5cf9f05ff6a1ee) --- .../Analysis/ScalarEvolutionPatternMatch.h | 14 +++++----- llvm/lib/Analysis/ScalarEvolution.cpp | 27 +++++++------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h index e30e94536993e..24a86c59d1cdb 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionPatternMatch.h @@ -95,6 +95,10 @@ inline bind_ty m_scev_Add(const SCEVAddExpr *&V) { return V; } +inline bind_ty m_scev_Mul(const SCEVMulExpr *&V) { + return V; +} + /// Match a specified const SCEV *. struct specificscev_ty { const SCEV *Expr; @@ -247,14 +251,10 @@ template struct SCEVURem_match { << SE.getTypeSizeInBits(TruncTy)); return Op0.match(LHS) && Op1.match(RHS); } - const auto *Add = dyn_cast(Expr); - if (Add == nullptr || Add->getNumOperands() != 2) - return false; - - const SCEV *A = Add->getOperand(1); - const auto *Mul = dyn_cast(Add->getOperand(0)); - if (Mul == nullptr) + const SCEV *A; + const SCEVMulExpr *Mul; + if (!SCEVPatternMatch::match(Expr, m_scev_Add(m_scev_Mul(Mul), m_SCEV(A)))) return false; const auto MatchURemWithDivisor = [&](const SCEV *B) { diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 1ee8a56d66f55..39f788bb435d4 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -4616,17 +4616,11 @@ const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, /// If Expr computes ~A, return A else return nullptr static const SCEV *MatchNotExpr(const SCEV *Expr) { - const SCEVAddExpr *Add = dyn_cast(Expr); - if (!Add || Add->getNumOperands() != 2 || - !Add->getOperand(0)->isAllOnesValue()) - return nullptr; - - const SCEVMulExpr *AddRHS = dyn_cast(Add->getOperand(1)); - if (!AddRHS || AddRHS->getNumOperands() != 2 || - !AddRHS->getOperand(0)->isAllOnesValue()) - return nullptr; - - return AddRHS->getOperand(1); + const SCEV *MulOp; + if (match(Expr, m_scev_Add(m_scev_AllOnes(), + m_scev_Mul(m_scev_AllOnes(), m_SCEV(MulOp))))) + return MulOp; + return nullptr; } /// Return a SCEV corresponding to ~V = -1-V @@ -12186,12 +12180,11 @@ ScalarEvolution::computeConstantDifference(const SCEV *More, const SCEV *Less) { // Try to match a common constant multiply. auto MatchConstMul = [](const SCEV *S) -> std::optional> { - auto *M = dyn_cast(S); - if (!M || M->getNumOperands() != 2 || - !isa(M->getOperand(0))) - return std::nullopt; - return { - {M->getOperand(1), cast(M->getOperand(0))->getAPInt()}}; + const APInt *C; + const SCEV *Op; + if (match(S, m_scev_Mul(m_scev_APInt(C), m_SCEV(Op)))) + return {{Op, *C}}; + return std::nullopt; }; if (auto MatchedMore = MatchConstMul(More)) { if (auto MatchedLess = MatchConstMul(Less)) { From 98e17c646ef9bc037ff7565ed97dd565296ed31e Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 18 Oct 2025 13:32:40 +0100 Subject: [PATCH 50/60] [SCEV] Rewrite A - B = UMin(1, A - B) lazily for A != B loop guards. (#163787) Follow-up to 2d027260b0f8 (https://github.com/llvm/llvm-project/pull/160500) Creating the SCEV subtraction eagerly is very expensive. To soften the blow, just collect a map with inequalities and check if we can apply the subtract rewrite when rewriting SCEVAddExpr. Restores most of the regression: http://llvm-compile-time-tracker.com/compare.php?from=0792478e4e133be96650444f3264e89d002fc058&to=7fca35db60fe6f423ea6051b45226046c067c252&stat=instructions:u stage1-O3: -0.10% stage1-ReleaseThinLTO: -0.09% stage1-ReleaseLTO-g: -0.10% stage1-O0-g: +0.02% stage1-aarch64-O3: -0.09% stage1-aarch64-O0-g: +0.00% stage2-O3: -0.17% stage2-O0-g: -0.05% stage2-clang: -0.07% There is still some negative impact compared to before 2d027260b0f8, but there's probably not much we could do reduce this even more. Compile-time improvement with 2d027260b0f8 reverted on top of the current PR: http://llvm-compile-time-tracker.com/compare.php?from=7fca35db60fe6f423ea6051b45226046c067c252&to=98dd152bdfc76b30d00190d3850d89406ca3c21f&stat=instructions:u stage1-O3: 60628M (-0.03%) stage1-ReleaseThinLTO: 76388M (-0.04%) stage1-ReleaseLTO-g: 89228M (-0.02%) stage1-O0-g: 18523M (-0.03%) stage1-aarch64-O3: 67623M (-0.03%) stage1-aarch64-O0-g: 22595M (+0.01%) stage2-O3: 52336M (+0.01%) stage2-O0-g: 16174M (+0.00%) stage2-clang: 34890032M (-0.03%) PR: https://github.com/llvm/llvm-project/pull/163787 (cherry picked from commit a5d3522c137bdaf270480dfe0e5b3fb977e4cc73) --- llvm/include/llvm/Analysis/ScalarEvolution.h | 1 + llvm/lib/Analysis/ScalarEvolution.cpp | 53 +++++-- .../IndVarSimplify/pointer-loop-guards.ll | 146 ++++++++++++++++++ 3 files changed, 189 insertions(+), 11 deletions(-) diff --git a/llvm/include/llvm/Analysis/ScalarEvolution.h b/llvm/include/llvm/Analysis/ScalarEvolution.h index 3b11fd86bdc99..ca0b7823558ff 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolution.h +++ b/llvm/include/llvm/Analysis/ScalarEvolution.h @@ -1343,6 +1343,7 @@ class ScalarEvolution { class LoopGuards { DenseMap RewriteMap; + SmallDenseSet> NotEqual; bool PreserveNUW = false; bool PreserveNSW = false; ScalarEvolution &SE; diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 39f788bb435d4..b2c3b5e817c86 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15699,19 +15699,26 @@ void ScalarEvolution::LoopGuards::collectFromBlock( GetNextSCEVDividesByDivisor(One, DividesBy); To = SE.getUMaxExpr(FromRewritten, OneAlignedUp); } else { + // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS), + // but creating the subtraction eagerly is expensive. Track the + // inequalities in a separate map, and materialize the rewrite lazily + // when encountering a suitable subtraction while re-writing. if (LHS->getType()->isPointerTy()) { LHS = SE.getLosslessPtrToIntExpr(LHS); RHS = SE.getLosslessPtrToIntExpr(RHS); if (isa(LHS) || isa(RHS)) break; } - auto AddSubRewrite = [&](const SCEV *A, const SCEV *B) { - const SCEV *Sub = SE.getMinusSCEV(A, B); - AddRewrite(Sub, Sub, - SE.getUMaxExpr(Sub, SE.getOne(From->getType()))); - }; - AddSubRewrite(LHS, RHS); - AddSubRewrite(RHS, LHS); + const SCEVConstant *C; + const SCEV *A, *B; + if (match(RHS, m_scev_Add(m_SCEVConstant(C), m_SCEV(A))) && + match(LHS, m_scev_Add(m_scev_Specific(C), m_SCEV(B)))) { + RHS = A; + LHS = B; + } + if (LHS > RHS) + std::swap(LHS, RHS); + Guards.NotEqual.insert({LHS, RHS}); continue; } break; @@ -15845,13 +15852,15 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const { class SCEVLoopGuardRewriter : public SCEVRewriteVisitor { const DenseMap ⤅ + const SmallDenseSet> ≠ SCEV::NoWrapFlags FlagMask = SCEV::FlagAnyWrap; public: SCEVLoopGuardRewriter(ScalarEvolution &SE, const ScalarEvolution::LoopGuards &Guards) - : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap) { + : SCEVRewriteVisitor(SE), Map(Guards.RewriteMap), + NotEqual(Guards.NotEqual) { if (Guards.PreserveNUW) FlagMask = ScalarEvolution::setFlags(FlagMask, SCEV::FlagNUW); if (Guards.PreserveNSW) @@ -15906,14 +15915,36 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const { } const SCEV *visitAddExpr(const SCEVAddExpr *Expr) { + // Helper to check if S is a subtraction (A - B) where A != B, and if so, + // return UMax(S, 1). + auto RewriteSubtraction = [&](const SCEV *S) -> const SCEV * { + const SCEV *LHS, *RHS; + if (MatchBinarySub(S, LHS, RHS)) { + if (LHS > RHS) + std::swap(LHS, RHS); + if (NotEqual.contains({LHS, RHS})) + return SE.getUMaxExpr(S, SE.getOne(S->getType())); + } + return nullptr; + }; + + // Check if Expr itself is a subtraction pattern with guard info. + if (const SCEV *Rewritten = RewriteSubtraction(Expr)) + return Rewritten; + // Trip count expressions sometimes consist of adding 3 operands, i.e. // (Const + A + B). There may be guard info for A + B, and if so, apply // it. // TODO: Could more generally apply guards to Add sub-expressions. if (isa(Expr->getOperand(0)) && Expr->getNumOperands() == 3) { - if (const SCEV *S = Map.lookup( - SE.getAddExpr(Expr->getOperand(1), Expr->getOperand(2)))) + const SCEV *Add = + SE.getAddExpr(Expr->getOperand(1), Expr->getOperand(2)); + if (const SCEV *Rewritten = RewriteSubtraction(Add)) + return SE.getAddExpr( + Expr->getOperand(0), Rewritten, + ScalarEvolution::maskFlags(Expr->getNoWrapFlags(), FlagMask)); + if (const SCEV *S = Map.lookup(Add)) return SE.getAddExpr(Expr->getOperand(0), S); } SmallVector Operands; @@ -15948,7 +15979,7 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const { } }; - if (RewriteMap.empty()) + if (RewriteMap.empty() && NotEqual.empty()) return Expr; SCEVLoopGuardRewriter Rewriter(SE, *this); diff --git a/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll b/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll index 9371fe24be05d..3bf1c2ef81cb9 100644 --- a/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll +++ b/llvm/test/Transforms/IndVarSimplify/pointer-loop-guards.ll @@ -48,3 +48,149 @@ exit: %res = phi i64 [ 0, %entry ], [ %i64.iv, %loop.latch ], [ 0, %loop.header ] ret i64 %res } + +define void @test_sub_cmp(ptr align 8 %start, ptr %end) { +; CHECK-LABEL: define void @test_sub_cmp( +; CHECK-SAME: ptr align 8 [[START:%.*]], ptr [[END:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[START_INT:%.*]] = ptrtoint ptr [[START]] to i64 +; CHECK-NEXT: [[END_INT:%.*]] = ptrtoint ptr [[END]] to i64 +; CHECK-NEXT: [[PTR_DIFF:%.*]] = sub i64 [[START_INT]], [[END_INT]] +; CHECK-NEXT: [[CMP_ENTRY:%.*]] = icmp eq ptr [[START]], [[END]] +; CHECK-NEXT: br i1 [[CMP_ENTRY]], label %[[EXIT:.*]], label %[[LOOP_HEADER_PREHEADER:.*]] +; CHECK: [[LOOP_HEADER_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP_HEADER:.*]] +; CHECK: [[LOOP_HEADER]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: [[C_1:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[C_1]], label %[[EXIT_EARLY:.*]], label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 +; CHECK-NEXT: [[CMP_LATCH:%.*]] = icmp ult i64 [[IV_NEXT]], [[PTR_DIFF]] +; CHECK-NEXT: br i1 [[CMP_LATCH]], label %[[LOOP_HEADER]], label %[[EXIT_LOOPEXIT:.*]] +; CHECK: [[EXIT_EARLY]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +; N32-LABEL: define void @test_sub_cmp( +; N32-SAME: ptr align 8 [[START:%.*]], ptr [[END:%.*]]) { +; N32-NEXT: [[ENTRY:.*:]] +; N32-NEXT: [[START_INT:%.*]] = ptrtoint ptr [[START]] to i64 +; N32-NEXT: [[END_INT:%.*]] = ptrtoint ptr [[END]] to i64 +; N32-NEXT: [[PTR_DIFF:%.*]] = sub i64 [[START_INT]], [[END_INT]] +; N32-NEXT: [[CMP_ENTRY:%.*]] = icmp eq ptr [[START]], [[END]] +; N32-NEXT: br i1 [[CMP_ENTRY]], label %[[EXIT:.*]], label %[[LOOP_HEADER_PREHEADER:.*]] +; N32: [[LOOP_HEADER_PREHEADER]]: +; N32-NEXT: br label %[[LOOP_HEADER:.*]] +; N32: [[LOOP_HEADER]]: +; N32-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ 0, %[[LOOP_HEADER_PREHEADER]] ] +; N32-NEXT: [[C_1:%.*]] = call i1 @cond() +; N32-NEXT: br i1 [[C_1]], label %[[EXIT_EARLY:.*]], label %[[LOOP_LATCH]] +; N32: [[LOOP_LATCH]]: +; N32-NEXT: [[IV_NEXT]] = add nuw i64 [[IV]], 1 +; N32-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[PTR_DIFF]] +; N32-NEXT: br i1 [[EXITCOND]], label %[[LOOP_HEADER]], label %[[EXIT_LOOPEXIT:.*]] +; N32: [[EXIT_EARLY]]: +; N32-NEXT: br label %[[EXIT]] +; N32: [[EXIT_LOOPEXIT]]: +; N32-NEXT: br label %[[EXIT]] +; N32: [[EXIT]]: +; N32-NEXT: ret void +; +entry: + %start.int = ptrtoint ptr %start to i64 + %end.int = ptrtoint ptr %end to i64 + %ptr.diff = sub i64 %start.int, %end.int + %cmp.entry = icmp eq ptr %start, %end + br i1 %cmp.entry, label %exit, label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %c.1 = call i1 @cond() + br i1 %c.1, label %exit.early, label %loop.latch + +loop.latch: + %iv.next = add i64 %iv, 1 + %cmp.latch = icmp ult i64 %iv.next, %ptr.diff + br i1 %cmp.latch, label %loop.header, label %exit + +exit.early: + br label %exit + +exit: + ret void +} + + +define void @test_ptr_diff_with_assume(ptr align 8 %start, ptr align 8 %end, ptr %P) { +; CHECK-LABEL: define void @test_ptr_diff_with_assume( +; CHECK-SAME: ptr align 8 [[START:%.*]], ptr align 8 [[END:%.*]], ptr [[P:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[START_INT:%.*]] = ptrtoint ptr [[START]] to i64 +; CHECK-NEXT: [[END_INT:%.*]] = ptrtoint ptr [[END]] to i64 +; CHECK-NEXT: [[PTR_DIFF:%.*]] = sub i64 [[START_INT]], [[END_INT]] +; CHECK-NEXT: [[DIFF_CMP:%.*]] = icmp ult i64 [[PTR_DIFF]], 2 +; CHECK-NEXT: call void @llvm.assume(i1 [[DIFF_CMP]]) +; CHECK-NEXT: [[COMPUTED_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[PTR_DIFF]] +; CHECK-NEXT: [[ENTRY_CMP:%.*]] = icmp eq ptr [[START]], [[END]] +; CHECK-NEXT: br i1 [[ENTRY_CMP]], label %[[EXIT:.*]], label %[[LOOP_BODY_PREHEADER:.*]] +; CHECK: [[LOOP_BODY_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP_BODY:.*]] +; CHECK: [[LOOP_BODY]]: +; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], %[[LOOP_BODY]] ], [ [[START]], %[[LOOP_BODY_PREHEADER]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i1 @cond() +; CHECK-NEXT: [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 1 +; CHECK-NEXT: [[LOOP_CMP:%.*]] = icmp eq ptr [[IV_NEXT]], [[COMPUTED_END]] +; CHECK-NEXT: br i1 [[LOOP_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP_BODY]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +; N32-LABEL: define void @test_ptr_diff_with_assume( +; N32-SAME: ptr align 8 [[START:%.*]], ptr align 8 [[END:%.*]], ptr [[P:%.*]]) { +; N32-NEXT: [[ENTRY:.*:]] +; N32-NEXT: [[START_INT:%.*]] = ptrtoint ptr [[START]] to i64 +; N32-NEXT: [[END_INT:%.*]] = ptrtoint ptr [[END]] to i64 +; N32-NEXT: [[PTR_DIFF:%.*]] = sub i64 [[START_INT]], [[END_INT]] +; N32-NEXT: [[DIFF_CMP:%.*]] = icmp ult i64 [[PTR_DIFF]], 2 +; N32-NEXT: call void @llvm.assume(i1 [[DIFF_CMP]]) +; N32-NEXT: [[COMPUTED_END:%.*]] = getelementptr i8, ptr [[START]], i64 [[PTR_DIFF]] +; N32-NEXT: [[ENTRY_CMP:%.*]] = icmp eq ptr [[START]], [[END]] +; N32-NEXT: br i1 [[ENTRY_CMP]], label %[[EXIT:.*]], label %[[LOOP_BODY_PREHEADER:.*]] +; N32: [[LOOP_BODY_PREHEADER]]: +; N32-NEXT: br label %[[LOOP_BODY:.*]] +; N32: [[LOOP_BODY]]: +; N32-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], %[[LOOP_BODY]] ], [ [[START]], %[[LOOP_BODY_PREHEADER]] ] +; N32-NEXT: [[TMP0:%.*]] = call i1 @cond() +; N32-NEXT: [[IV_NEXT]] = getelementptr i8, ptr [[IV]], i64 1 +; N32-NEXT: [[LOOP_CMP:%.*]] = icmp eq ptr [[IV_NEXT]], [[COMPUTED_END]] +; N32-NEXT: br i1 [[LOOP_CMP]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP_BODY]] +; N32: [[EXIT_LOOPEXIT]]: +; N32-NEXT: br label %[[EXIT]] +; N32: [[EXIT]]: +; N32-NEXT: ret void +; +entry: + %start.int = ptrtoint ptr %start to i64 + %end.int = ptrtoint ptr %end to i64 + %ptr.diff = sub i64 %start.int, %end.int + %diff.cmp = icmp ult i64 %ptr.diff, 2 + call void @llvm.assume(i1 %diff.cmp) + %computed.end = getelementptr i8, ptr %start, i64 %ptr.diff + %entry.cmp = icmp eq ptr %start, %end + br i1 %entry.cmp, label %exit, label %loop.body + +loop.body: + %iv = phi ptr [ %start, %entry ], [ %iv.next, %loop.body ] + call i1 @cond() + %iv.next = getelementptr i8, ptr %iv, i64 1 + %loop.cmp = icmp eq ptr %iv.next, %computed.end + br i1 %loop.cmp, label %exit, label %loop.body + +exit: + ret void +} From ba69e1dd68b4e5506c924120a202142d6dcd19fd Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 20 Oct 2025 10:20:41 +0100 Subject: [PATCH 51/60] [SCEV] Preserve divisor info when adding guard info for ICMP_NE via Sub. (#163250) Follow-up to https://github.com/llvm/llvm-project/pull/160500 to preserve divisibiltiy info when creating the UMax. PR: https://github.com/llvm/llvm-project/pull/163250 (cherry picked from commit eb17a8d599dbcadecba2529dcf9ac234b085088a) --- llvm/lib/Analysis/ScalarEvolution.cpp | 48 ++-- .../max-backedge-taken-count-guard-info.ll | 8 +- llvm/test/Transforms/LoopUnroll/scevunroll.ll | 12 +- .../single-early-exit-deref-assumptions.ll | 237 ++++++++++++++++++ 4 files changed, 272 insertions(+), 33 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index b2c3b5e817c86..8b9fea89c1f2a 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15426,6 +15426,23 @@ void ScalarEvolution::LoopGuards::collectFromPHI( } } +// Return a new SCEV that modifies \p Expr to the closest number divides by +// \p Divisor and greater or equal than Expr. For now, only handle constant +// Expr. +static const SCEV *getNextSCEVDividesByDivisor(const SCEV *Expr, + const APInt &DivisorVal, + ScalarEvolution &SE) { + const APInt *ExprVal; + if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || + DivisorVal.isNonPositive()) + return Expr; + APInt Rem = ExprVal->urem(DivisorVal); + if (Rem.isZero()) + return Expr; + // return the SCEV: Expr + Divisor - Expr % Divisor + return SE.getConstant(*ExprVal + DivisorVal - Rem); +} + void ScalarEvolution::LoopGuards::collectFromBlock( ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards, const BasicBlock *Block, const BasicBlock *Pred, @@ -15499,22 +15516,6 @@ void ScalarEvolution::LoopGuards::collectFromBlock( return false; }; - // Return a new SCEV that modifies \p Expr to the closest number divides by - // \p Divisor and greater or equal than Expr. For now, only handle constant - // Expr. - auto GetNextSCEVDividesByDivisor = [&](const SCEV *Expr, - const APInt &DivisorVal) { - const APInt *ExprVal; - if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || - DivisorVal.isNonPositive()) - return Expr; - APInt Rem = ExprVal->urem(DivisorVal); - if (Rem.isZero()) - return Expr; - // return the SCEV: Expr + Divisor - Expr % Divisor - return SE.getConstant(*ExprVal + DivisorVal - Rem); - }; - // Return a new SCEV that modifies \p Expr to the closest number divides by // \p Divisor and less or equal than Expr. For now, only handle constant // Expr. @@ -15551,7 +15552,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( "Expected non-negative operand!"); auto *DivisibleExpr = IsMin ? GetPreviousSCEVDividesByDivisor(MinMaxLHS, DivisorVal) - : GetNextSCEVDividesByDivisor(MinMaxLHS, DivisorVal); + : getNextSCEVDividesByDivisor(MinMaxLHS, DivisorVal, SE); SmallVector Ops = { ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr}; return SE.getMinMaxExpr(SCTy, Ops); @@ -15634,7 +15635,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( case CmpInst::ICMP_UGT: case CmpInst::ICMP_SGT: RHS = SE.getAddExpr(RHS, One); - RHS = GetNextSCEVDividesByDivisor(RHS, DividesBy); + RHS = getNextSCEVDividesByDivisor(RHS, DividesBy, SE); break; case CmpInst::ICMP_ULE: case CmpInst::ICMP_SLE: @@ -15642,7 +15643,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( break; case CmpInst::ICMP_UGE: case CmpInst::ICMP_SGE: - RHS = GetNextSCEVDividesByDivisor(RHS, DividesBy); + RHS = getNextSCEVDividesByDivisor(RHS, DividesBy, SE); break; default: break; @@ -15696,7 +15697,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( case CmpInst::ICMP_NE: if (match(RHS, m_scev_Zero())) { const SCEV *OneAlignedUp = - GetNextSCEVDividesByDivisor(One, DividesBy); + getNextSCEVDividesByDivisor(One, DividesBy, SE); To = SE.getUMaxExpr(FromRewritten, OneAlignedUp); } else { // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS), @@ -15922,8 +15923,11 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const { if (MatchBinarySub(S, LHS, RHS)) { if (LHS > RHS) std::swap(LHS, RHS); - if (NotEqual.contains({LHS, RHS})) - return SE.getUMaxExpr(S, SE.getOne(S->getType())); + if (NotEqual.contains({LHS, RHS})) { + const SCEV *OneAlignedUp = getNextSCEVDividesByDivisor( + SE.getOne(S->getType()), SE.getConstantMultiple(S), SE); + return SE.getUMaxExpr(OneAlignedUp, S); + } } return nullptr; }; diff --git a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll index 4024c986dd11d..d5a2181e9bc5e 100644 --- a/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/max-backedge-taken-count-guard-info.ll @@ -1431,7 +1431,7 @@ define void @ptr_induction_early_exit_eq_1_with_align_on_load(ptr %a, ptr %b, pt ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693951 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693950 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) @@ -1470,7 +1470,7 @@ define void @ptr_induction_early_exit_eq_1_with_align_on_arguments(ptr align 8 % ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693951 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693950 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) /u 8) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a to i64)) + (ptrtoint ptr %b to i64)) /u 8) @@ -1511,7 +1511,7 @@ define void @ptr_induction_early_exit_eq_1_align_assumption_1(ptr %a, ptr %b, pt ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693951 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693950 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) @@ -1556,7 +1556,7 @@ define void @ptr_induction_early_exit_eq_1_align_assumption_2(ptr %a, ptr %b, pt ; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. ; CHECK-NEXT: exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) -; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693951 +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is i64 2305843009213693950 ; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) ; CHECK-NEXT: symbolic max exit count for loop: ***COULDNOTCOMPUTE*** ; CHECK-NEXT: symbolic max exit count for loop.inc: ((-8 + (-1 * (ptrtoint ptr %a_ to i64)) + (ptrtoint ptr %b_ to i64)) /u 8) diff --git a/llvm/test/Transforms/LoopUnroll/scevunroll.ll b/llvm/test/Transforms/LoopUnroll/scevunroll.ll index a3cd07248c54b..995eb0c52ff36 100644 --- a/llvm/test/Transforms/LoopUnroll/scevunroll.ll +++ b/llvm/test/Transforms/LoopUnroll/scevunroll.ll @@ -439,20 +439,18 @@ declare void @fn(i32) define void @peel_int_eq_condition(i32 %start) { ; CHECK-LABEL: @peel_int_eq_condition( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[SMAX:%.*]] = call i32 @llvm.smax.i32(i32 [[START:%.*]], i32 100) -; CHECK-NEXT: [[TMP0:%.*]] = add nuw i32 [[SMAX]], 1 ; CHECK-NEXT: br label [[LOOP_PEEL_BEGIN:%.*]] ; CHECK: loop.peel.begin: ; CHECK-NEXT: br label [[LOOP_PEEL:%.*]] ; CHECK: loop.peel: -; CHECK-NEXT: [[C_0_PEEL:%.*]] = icmp eq i32 [[START]], [[START]] +; CHECK-NEXT: [[C_0_PEEL:%.*]] = icmp eq i32 [[START:%.*]], [[START]] ; CHECK-NEXT: br i1 [[C_0_PEEL]], label [[IF_THEN_PEEL:%.*]], label [[LOOP_LATCH_PEEL:%.*]] ; CHECK: if.then.peel: ; CHECK-NEXT: call void @fn(i32 [[START]]) ; CHECK-NEXT: br label [[LOOP_LATCH_PEEL]] ; CHECK: loop.latch.peel: ; CHECK-NEXT: [[IV_NEXT_PEEL:%.*]] = add i32 [[START]], 1 -; CHECK-NEXT: [[EXITCOND_PEEL:%.*]] = icmp ne i32 [[IV_NEXT_PEEL]], [[TMP0]] +; CHECK-NEXT: [[EXITCOND_PEEL:%.*]] = icmp slt i32 [[START]], 100 ; CHECK-NEXT: br i1 [[EXITCOND_PEEL]], label [[LOOP_PEEL_NEXT:%.*]], label [[EXIT:%.*]] ; CHECK: loop.peel.next: ; CHECK-NEXT: br label [[LOOP_PEEL_NEXT1:%.*]] @@ -467,9 +465,9 @@ define void @peel_int_eq_condition(i32 %start) { ; CHECK-NEXT: call void @fn(i32 [[IV]]) ; CHECK-NEXT: br label [[LOOP_LATCH]] ; CHECK: loop.latch: -; CHECK-NEXT: [[IV_NEXT]] = add i32 [[IV]], 1 -; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i32 [[IV_NEXT]], [[TMP0]] -; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: [[IV_NEXT]] = add nsw i32 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp slt i32 [[IV]], 100 +; CHECK-NEXT: br i1 [[EC]], label [[LOOP]], label [[EXIT_LOOPEXIT:%.*]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: exit.loopexit: ; CHECK-NEXT: br label [[EXIT]] ; CHECK: exit: diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index a07e0069efbc4..b5b8f6ea14cfc 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -406,3 +406,240 @@ exit: %res = ptrtoint ptr %p to i64 ret i64 %res } + +; The existing assumptions is strong enough to vectorize this. +define ptr @find_deref_pointer_distance_align_attribute_argument(ptr align 2 %first, ptr align 2 %last) nofree nosync { +; CHECK-LABEL: define ptr @find_deref_pointer_distance_align_attribute_argument( +; CHECK-SAME: ptr align 2 [[FIRST:%.*]], ptr align 2 [[LAST:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[FIRST]], i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[LAST]], i64 2) ] +; CHECK-NEXT: [[LAST_I64:%.*]] = ptrtoint ptr [[LAST]] to i64 +; CHECK-NEXT: [[FIRST_I64:%.*]] = ptrtoint ptr [[FIRST]] to i64 +; CHECK-NEXT: [[DISTANCE:%.*]] = sub i64 [[LAST_I64]], [[FIRST_I64]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[FIRST]], i64 [[DISTANCE]]) ] +; CHECK-NEXT: [[C_0:%.*]] = icmp eq ptr [[FIRST]], [[LAST]] +; CHECK-NEXT: br i1 [[C_0]], label [[EXIT:%.*]], label [[LOOP_HEADER_PREHEADER:%.*]] +; CHECK: loop.header.preheader: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[LAST_I64]], -2 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[FIRST_I64]] +; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[TMP4]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP15]], align 2 +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD]], splat (i16 1) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] +; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP8]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true) +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[TMP13]] +; CHECK-NEXT: br label [[EXIT_LOOPEXIT]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ [[FIRST]], [[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[IV]], align 2 +; CHECK-NEXT: [[C_1:%.*]] = icmp eq i16 [[L]], 1 +; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[IV]], i64 2 +; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[IV_NEXT]], [[LAST]] +; CHECK-NEXT: br i1 [[C_2]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: exit.loopexit: +; CHECK-NEXT: [[FIRST_ADDR_0_LCSSA_I_PH:%.*]] = phi ptr [ [[IV_NEXT]], [[LOOP_LATCH]] ], [ [[IV]], [[LOOP_HEADER]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ [[TMP14]], [[VECTOR_EARLY_EXIT]] ] +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: [[FIRST_ADDR_0_LCSSA_I:%.*]] = phi ptr [ [[FIRST]], [[ENTRY:%.*]] ], [ [[FIRST_ADDR_0_LCSSA_I_PH]], [[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret ptr [[FIRST_ADDR_0_LCSSA_I]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %first, i64 2) ] + call void @llvm.assume(i1 true) [ "align"(ptr %last, i64 2) ] + %last.i64 = ptrtoint ptr %last to i64 + %first.i64 = ptrtoint ptr %first to i64 + %distance = sub i64 %last.i64, %first.i64 + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %first, i64 %distance) ] + %c.0 = icmp eq ptr %first, %last + br i1 %c.0, label %exit, label %loop.header + +loop.header: + %iv = phi ptr [ %first, %entry ], [ %iv.next, %loop.latch ] + %l = load i16, ptr %iv, align 2 + %c.1 = icmp eq i16 %l, 1 + br i1 %c.1, label %exit, label %loop.latch + +loop.latch: + %iv.next = getelementptr inbounds nuw i8, ptr %iv, i64 2 + %c.2 = icmp eq ptr %iv.next, %last + br i1 %c.2, label %exit, label %loop.header + +exit: + %first.addr.0.lcssa.i = phi ptr [ %first, %entry ], [ %iv, %loop.header ], [ %iv.next, %loop.latch ] + ret ptr %first.addr.0.lcssa.i +} + +; The existing assumptions is strong enough to vectorize this. +define ptr @find_deref_pointer_distance_align_assumption(ptr %first, ptr %last) nofree nosync { +; CHECK-LABEL: define ptr @find_deref_pointer_distance_align_assumption( +; CHECK-SAME: ptr [[FIRST:%.*]], ptr [[LAST:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[LAST_I64:%.*]] = ptrtoint ptr [[LAST]] to i64 +; CHECK-NEXT: [[FIRST_I64:%.*]] = ptrtoint ptr [[FIRST]] to i64 +; CHECK-NEXT: [[DISTANCE:%.*]] = sub i64 [[LAST_I64]], [[FIRST_I64]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[FIRST]], i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[LAST]], i64 2) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[FIRST]], i64 [[DISTANCE]]) ] +; CHECK-NEXT: [[C_0:%.*]] = icmp eq ptr [[FIRST]], [[LAST]] +; CHECK-NEXT: br i1 [[C_0]], label [[EXIT:%.*]], label [[LOOP_HEADER_PREHEADER:%.*]] +; CHECK: loop.header.preheader: +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[LAST_I64]], -2 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[FIRST_I64]] +; CHECK-NEXT: [[TMP2:%.*]] = lshr i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw i64 [[TMP2]], 1 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[TMP3]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], 4 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[TMP4]] +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 2 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i16, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i16>, ptr [[TMP15]], align 2 +; CHECK-NEXT: [[TMP6:%.*]] = icmp eq <4 x i16> [[WIDE_LOAD]], splat (i16 1) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP7:%.*]] = freeze <4 x i1> [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP7]]) +; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: [[TMP10:%.*]] = or i1 [[TMP8]], [[TMP9]] +; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP8]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[TMP3]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP6]], i1 true) +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[FIRST]], i64 [[TMP13]] +; CHECK-NEXT: br label [[EXIT_LOOPEXIT]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ [[FIRST]], [[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: [[IV:%.*]] = phi ptr [ [[IV_NEXT:%.*]], [[LOOP_LATCH:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[L:%.*]] = load i16, ptr [[IV]], align 2 +; CHECK-NEXT: [[C_1:%.*]] = icmp eq i16 [[L]], 1 +; CHECK-NEXT: br i1 [[C_1]], label [[EXIT_LOOPEXIT]], label [[LOOP_LATCH]] +; CHECK: loop.latch: +; CHECK-NEXT: [[IV_NEXT]] = getelementptr inbounds nuw i8, ptr [[IV]], i64 2 +; CHECK-NEXT: [[C_2:%.*]] = icmp eq ptr [[IV_NEXT]], [[LAST]] +; CHECK-NEXT: br i1 [[C_2]], label [[EXIT_LOOPEXIT]], label [[LOOP_HEADER]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: exit.loopexit: +; CHECK-NEXT: [[FIRST_ADDR_0_LCSSA_I_PH:%.*]] = phi ptr [ [[IV_NEXT]], [[LOOP_LATCH]] ], [ [[IV]], [[LOOP_HEADER]] ], [ [[TMP5]], [[MIDDLE_BLOCK]] ], [ [[TMP14]], [[VECTOR_EARLY_EXIT]] ] +; CHECK-NEXT: br label [[EXIT]] +; CHECK: exit: +; CHECK-NEXT: [[FIRST_ADDR_0_LCSSA_I:%.*]] = phi ptr [ [[FIRST]], [[ENTRY:%.*]] ], [ [[FIRST_ADDR_0_LCSSA_I_PH]], [[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret ptr [[FIRST_ADDR_0_LCSSA_I]] +; +entry: + %last.i64 = ptrtoint ptr %last to i64 + %first.i64 = ptrtoint ptr %first to i64 + %distance = sub i64 %last.i64, %first.i64 + call void @llvm.assume(i1 true) [ "align"(ptr %first, i64 2) ] + call void @llvm.assume(i1 true) [ "align"(ptr %last, i64 2) ] + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %first, i64 %distance) ] + %c.0 = icmp eq ptr %first, %last + br i1 %c.0, label %exit, label %loop.header + +loop.header: + %iv = phi ptr [ %first, %entry ], [ %iv.next, %loop.latch ] + %l = load i16, ptr %iv, align 2 + %c.1 = icmp eq i16 %l, 1 + br i1 %c.1, label %exit, label %loop.latch + +loop.latch: + %iv.next = getelementptr inbounds nuw i8, ptr %iv, i64 2 + %c.2 = icmp eq ptr %iv.next, %last + br i1 %c.2, label %exit, label %loop.header + +exit: + %first.addr.0.lcssa.i = phi ptr [ %first, %entry ], [ %iv, %loop.header ], [ %iv.next, %loop.latch ] + ret ptr %first.addr.0.lcssa.i +} + +define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size_nofree_via_context(ptr noalias %p1, ptr noalias %p2) nosync { +; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size_nofree_via_context( +; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ] +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX1]] +; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1 +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX1]] +; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[TMP1]], align 1 +; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] +; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]] +; CHECK: loop.inc: +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX1]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024 +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]] +; CHECK: loop.end: +; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX1]], [[LOOP]] ], [ -1, [[LOOP_INC]] ] +; CHECK-NEXT: ret i64 [[RETVAL]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %p1, i64 4), "dereferenceable"(ptr %p1, i64 1024) ] + call void @llvm.assume(i1 true) [ "align"(ptr %p2, i64 4), "dereferenceable"(ptr %p2, i64 1024) ] + br label %loop + +loop: + %index = phi i64 [ %index.next, %loop.inc ], [ 0, %entry ] + %arrayidx = getelementptr inbounds i8, ptr %p1, i64 %index + %ld1 = load i8, ptr %arrayidx, align 1 + %arrayidx1 = getelementptr inbounds i8, ptr %p2, i64 %index + %ld2 = load i8, ptr %arrayidx1, align 1 + %cmp3 = icmp eq i8 %ld1, %ld2 + br i1 %cmp3, label %loop.inc, label %loop.end + +loop.inc: + %index.next = add i64 %index, 1 + %exitcond = icmp ne i64 %index.next, 1024 + br i1 %exitcond, label %loop, label %loop.end + +loop.end: + %retval = phi i64 [ %index, %loop ], [ -1, %loop.inc ] + ret i64 %retval +} From ea3b9c083524b83876f3dafd9ccbc46beff21237 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 20 Oct 2025 14:30:06 +0100 Subject: [PATCH 52/60] [SCEV] Add extra test coverage with URem & AddRec guards. Add test with urem guard with non-constant divisor and AddRec guards. Extra test coverage for https://github.com/llvm/llvm-project/pull/163021 (cherry picked from commit 0731f18ba9caee809bca43a67d0a4893a147011e) --- .../backedge-taken-count-guard-info.ll | 53 +++++++++++++++++++ .../trip-multiple-guard-info.ll | 41 ++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll index 3c3748a6a5f02..1964fca603e23 100644 --- a/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/backedge-taken-count-guard-info.ll @@ -104,3 +104,56 @@ exit: } declare void @clobber() + + +declare void @clobber.i32(i32) + +define void @test_guards_across_loops(i32 %N) { +; CHECK-LABEL: 'test_guards_across_loops' +; CHECK-NEXT: Classifying expressions for: @test_guards_across_loops +; CHECK-NEXT: %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.1 ] +; CHECK-NEXT: --> {0,+,1}<%loop.1> U: full-set S: full-set Exits: <> LoopDispositions: { %loop.1: Computable } +; CHECK-NEXT: %iv.1.next = add i32 %iv.1, 1 +; CHECK-NEXT: --> {1,+,1}<%loop.1> U: full-set S: full-set Exits: <> LoopDispositions: { %loop.1: Computable } +; CHECK-NEXT: %iv.2 = phi i32 [ 0, %loop.1 ], [ %iv.2.next, %loop.2 ] +; CHECK-NEXT: --> {0,+,1}<%loop.2> U: full-set S: full-set Exits: (1 + %N) LoopDispositions: { %loop.2: Computable } +; CHECK-NEXT: %iv.2.next = add i32 %iv.2, 1 +; CHECK-NEXT: --> {1,+,1}<%loop.2> U: full-set S: full-set Exits: (2 + %N) LoopDispositions: { %loop.2: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_guards_across_loops +; CHECK-NEXT: Loop %loop.2: backedge-taken count is (1 + (zext i32 %N to i64)) +; CHECK-NEXT: Loop %loop.2: constant max backedge-taken count is i64 4294967296 +; CHECK-NEXT: Loop %loop.2: symbolic max backedge-taken count is (1 + (zext i32 %N to i64)) +; CHECK-NEXT: Loop %loop.2: Trip multiple is 1 +; CHECK-NEXT: Loop %loop.1: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop.1: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop.1: Unpredictable symbolic max backedge-taken count. +; CHECK-NEXT: Loop %loop.1: Predicated backedge-taken count is (1 + (zext i32 %N to i64)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: {0,+,1}<%loop.1> Added Flags: +; CHECK-NEXT: Loop %loop.1: Predicated constant max backedge-taken count is i64 4294967296 +; CHECK-NEXT: Predicates: +; CHECK-NEXT: {0,+,1}<%loop.1> Added Flags: +; CHECK-NEXT: Loop %loop.1: Predicated symbolic max backedge-taken count is (1 + (zext i32 %N to i64)) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: {0,+,1}<%loop.1> Added Flags: +; +entry: + br label %loop.1 + +loop.1: + %iv.1 = phi i32 [ 0, %entry ], [ %iv.1.next, %loop.1 ] + call void @clobber.i32(i32 %iv.1) + %ec.1 = icmp ugt i32 %iv.1, %N + %iv.1.next = add i32 %iv.1, 1 + br i1 %ec.1, label %loop.2, label %loop.1 + +loop.2: + %iv.2 = phi i32 [ 0, %loop.1 ], [ %iv.2.next, %loop.2 ] + call void @clobber.i32(i32 %iv.2) + %ec.2 = icmp ugt i32 %iv.2, %N + %iv.2.next = add i32 %iv.2, 1 + br i1 %ec.2, label %exit, label %loop.2 + +exit: + ret void +} diff --git a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll index a477465cb0699..f35c48b3e6fc5 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-multiple-guard-info.ll @@ -717,5 +717,46 @@ exit: ret void } +define void @test_urem_non_constant(ptr %dst, i32 %a, i32 %b) { +; CHECK-LABEL: 'test_urem_non_constant' +; CHECK-NEXT: Classifying expressions for: @test_urem_non_constant +; CHECK-NEXT: %rem = urem i32 %a, %b +; CHECK-NEXT: --> ((-1 * (%a /u %b) * %b) + %a) U: full-set S: full-set +; CHECK-NEXT: %and.0 = and i1 %pre.0, %pre.1 +; CHECK-NEXT: --> (%pre.1 umin %pre.0) U: full-set S: full-set +; CHECK-NEXT: %and.1 = and i1 %and.0, %pre.2 +; CHECK-NEXT: --> (%pre.1 umin %pre.2 umin %pre.0) U: full-set S: full-set +; CHECK-NEXT: %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ] +; CHECK-NEXT: --> {0,+,%b}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: %gep.dst = getelementptr inbounds i8, ptr %dst, i32 %b +; CHECK-NEXT: --> ((sext i32 %b to i64) + %dst) U: full-set S: full-set Exits: ((sext i32 %b to i64) + %dst) LoopDispositions: { %loop: Invariant } +; CHECK-NEXT: %iv.next = add i32 %iv, %b +; CHECK-NEXT: --> {%b,+,%b}<%loop> U: full-set S: full-set Exits: <> LoopDispositions: { %loop: Computable } +; CHECK-NEXT: Determining loop execution counts for: @test_urem_non_constant +; CHECK-NEXT: Loop %loop: Unpredictable backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable constant max backedge-taken count. +; CHECK-NEXT: Loop %loop: Unpredictable symbolic max backedge-taken count. +; +entry: + %rem = urem i32 %a, %b + %pre.0 = icmp eq i32 %rem, 0 + %pre.1 = icmp ne i32 %a, 0 + %pre.2 = icmp ne i32 %b, 0 + %and.0 = and i1 %pre.0, %pre.1 + %and.1 = and i1 %and.0, %pre.2 + br i1 %and.1, label %loop, label %exit + +loop: + %iv = phi i32 [ 0, %entry], [ %iv.next, %loop ] + %gep.dst = getelementptr inbounds i8, ptr %dst, i32 %b + store i8 0, ptr %gep.dst + %iv.next = add i32 %iv, %b + %ec = icmp ne i32 %iv.next, %a + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + declare void @llvm.assume(i1) declare void @llvm.experimental.guard(i1, ...) From f2ef597bf33e4a6890a6c864f71f8e4bfdb11485 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 20 Oct 2025 15:04:54 +0100 Subject: [PATCH 53/60] [SCEV] Move and clarify names of prev/next divisor helpers (NFC). Move getPreviousSCEVDivisibleByDivisor from a lambda to a static function and clarify the name (DividesBy -> DivisibleBy). Split off refactoring from https://github.com/llvm/llvm-project/pull/163021. (cherry picked from commit 385ea0dbc132300ee6c716f77ad1de42918687d3) --- llvm/lib/Analysis/ScalarEvolution.cpp | 52 ++++++++++++++------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 8b9fea89c1f2a..a1ec13f8e2c60 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15426,12 +15426,27 @@ void ScalarEvolution::LoopGuards::collectFromPHI( } } +// Return a new SCEV that modifies \p Expr to the closest number divides by +// \p Divisor and less or equal than Expr. For now, only handle constant +// Expr. +static const SCEV *getPreviousSCEVDivisibleByDivisor(const SCEV *Expr, + const APInt &DivisorVal, + ScalarEvolution &SE) { + const APInt *ExprVal; + if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || + DivisorVal.isNonPositive()) + return Expr; + APInt Rem = ExprVal->urem(DivisorVal); + // return the SCEV: Expr - Expr % Divisor + return SE.getConstant(*ExprVal - Rem); +} + // Return a new SCEV that modifies \p Expr to the closest number divides by // \p Divisor and greater or equal than Expr. For now, only handle constant // Expr. -static const SCEV *getNextSCEVDividesByDivisor(const SCEV *Expr, - const APInt &DivisorVal, - ScalarEvolution &SE) { +static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr, + const APInt &DivisorVal, + ScalarEvolution &SE) { const APInt *ExprVal; if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || DivisorVal.isNonPositive()) @@ -15516,20 +15531,6 @@ void ScalarEvolution::LoopGuards::collectFromBlock( return false; }; - // Return a new SCEV that modifies \p Expr to the closest number divides by - // \p Divisor and less or equal than Expr. For now, only handle constant - // Expr. - auto GetPreviousSCEVDividesByDivisor = [&](const SCEV *Expr, - const APInt &DivisorVal) { - const APInt *ExprVal; - if (!match(Expr, m_scev_APInt(ExprVal)) || ExprVal->isNegative() || - DivisorVal.isNonPositive()) - return Expr; - APInt Rem = ExprVal->urem(DivisorVal); - // return the SCEV: Expr - Expr % Divisor - return SE.getConstant(*ExprVal - Rem); - }; - // Apply divisibilty by \p Divisor on MinMaxExpr with constant values, // recursively. This is done by aligning up/down the constant value to the // Divisor. @@ -15551,8 +15552,9 @@ void ScalarEvolution::LoopGuards::collectFromBlock( assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!"); auto *DivisibleExpr = - IsMin ? GetPreviousSCEVDividesByDivisor(MinMaxLHS, DivisorVal) - : getNextSCEVDividesByDivisor(MinMaxLHS, DivisorVal, SE); + IsMin + ? getPreviousSCEVDivisibleByDivisor(MinMaxLHS, DivisorVal, SE) + : getNextSCEVDivisibleByDivisor(MinMaxLHS, DivisorVal, SE); SmallVector Ops = { ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr}; return SE.getMinMaxExpr(SCTy, Ops); @@ -15629,21 +15631,21 @@ void ScalarEvolution::LoopGuards::collectFromBlock( [[fallthrough]]; case CmpInst::ICMP_SLT: { RHS = SE.getMinusSCEV(RHS, One); - RHS = GetPreviousSCEVDividesByDivisor(RHS, DividesBy); + RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE); break; } case CmpInst::ICMP_UGT: case CmpInst::ICMP_SGT: RHS = SE.getAddExpr(RHS, One); - RHS = getNextSCEVDividesByDivisor(RHS, DividesBy, SE); + RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE); break; case CmpInst::ICMP_ULE: case CmpInst::ICMP_SLE: - RHS = GetPreviousSCEVDividesByDivisor(RHS, DividesBy); + RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE); break; case CmpInst::ICMP_UGE: case CmpInst::ICMP_SGE: - RHS = getNextSCEVDividesByDivisor(RHS, DividesBy, SE); + RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE); break; default: break; @@ -15697,7 +15699,7 @@ void ScalarEvolution::LoopGuards::collectFromBlock( case CmpInst::ICMP_NE: if (match(RHS, m_scev_Zero())) { const SCEV *OneAlignedUp = - getNextSCEVDividesByDivisor(One, DividesBy, SE); + getNextSCEVDivisibleByDivisor(One, DividesBy, SE); To = SE.getUMaxExpr(FromRewritten, OneAlignedUp); } else { // LHS != RHS can be rewritten as (LHS - RHS) = UMax(1, LHS - RHS), @@ -15924,7 +15926,7 @@ const SCEV *ScalarEvolution::LoopGuards::rewrite(const SCEV *Expr) const { if (LHS > RHS) std::swap(LHS, RHS); if (NotEqual.contains({LHS, RHS})) { - const SCEV *OneAlignedUp = getNextSCEVDividesByDivisor( + const SCEV *OneAlignedUp = getNextSCEVDivisibleByDivisor( SE.getOne(S->getType()), SE.getConstantMultiple(S), SE); return SE.getUMaxExpr(OneAlignedUp, S); } From a738a80d5f0cbd920871f78994de5ecb69c16168 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 31 Oct 2025 11:27:52 -0700 Subject: [PATCH 54/60] [SCEV] Fix switch formatting in collectFromBlock (NFC). Fix formatting for switch, to avoid unrelated changes/formatting errors in https://github.com/llvm/llvm-project/pull/163021. (cherry picked from commit 817b7c5e562b34e861286fd781d6b422c7fa63bf) --- llvm/lib/Analysis/ScalarEvolution.cpp | 50 +++++++++++++-------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index a1ec13f8e2c60..9d54f305bec9d 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15624,31 +15624,31 @@ void ScalarEvolution::LoopGuards::collectFromBlock( // predicate. const SCEV *One = SE.getOne(RHS->getType()); switch (Predicate) { - case CmpInst::ICMP_ULT: - if (RHS->getType()->isPointerTy()) - return; - RHS = SE.getUMaxExpr(RHS, One); - [[fallthrough]]; - case CmpInst::ICMP_SLT: { - RHS = SE.getMinusSCEV(RHS, One); - RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE); - break; - } - case CmpInst::ICMP_UGT: - case CmpInst::ICMP_SGT: - RHS = SE.getAddExpr(RHS, One); - RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE); - break; - case CmpInst::ICMP_ULE: - case CmpInst::ICMP_SLE: - RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE); - break; - case CmpInst::ICMP_UGE: - case CmpInst::ICMP_SGE: - RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE); - break; - default: - break; + case CmpInst::ICMP_ULT: + if (RHS->getType()->isPointerTy()) + return; + RHS = SE.getUMaxExpr(RHS, One); + [[fallthrough]]; + case CmpInst::ICMP_SLT: { + RHS = SE.getMinusSCEV(RHS, One); + RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE); + break; + } + case CmpInst::ICMP_UGT: + case CmpInst::ICMP_SGT: + RHS = SE.getAddExpr(RHS, One); + RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE); + break; + case CmpInst::ICMP_ULE: + case CmpInst::ICMP_SLE: + RHS = getPreviousSCEVDivisibleByDivisor(RHS, DividesBy, SE); + break; + case CmpInst::ICMP_UGE: + case CmpInst::ICMP_SGE: + RHS = getNextSCEVDivisibleByDivisor(RHS, DividesBy, SE); + break; + default: + break; } SmallVector Worklist(1, LHS); From 208a5b2435c275ec372f2e7e58708d33ce283c43 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 2 Nov 2025 14:16:24 +0000 Subject: [PATCH 55/60] [SCEV] Improve handling of divisibility information from loop guards. (#163021) At the moment, the effectivness of guards that contain divisibility information (A % B == 0 ) depends on the order of the conditions. This patch makes using divisibility information independent of the order, by collecting and applying the divisibility information separately. We first collect all conditions in a vector, then collect the divisibility information from all guards. When processing other guards, we apply divisibility info collected earlier. After all guards have been processed, we add the divisibility info, rewriting the existing rewrite. This ensures we apply the divisibility info to the largest rewrite expression. This helps to improve results in a few cases, one in https://github.com/dtcxzyw/llvm-opt-benchmark/pull/2921 and another one in a different large C/C++ based IR corpus. PR: https://github.com/llvm/llvm-project/pull/163021 (cherry picked from commit d3fe1df1945a6761e042ea02b7a80154d648601a) --- llvm/lib/Analysis/ScalarEvolution.cpp | 195 ++++++++++------- .../IndVarSimplify/loop-guard-order.ll | 198 ++++++++++++++++++ 2 files changed, 312 insertions(+), 81 deletions(-) diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index 9d54f305bec9d..239849e670350 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15458,6 +15458,78 @@ static const SCEV *getNextSCEVDivisibleByDivisor(const SCEV *Expr, return SE.getConstant(*ExprVal + DivisorVal - Rem); } +static bool collectDivisibilityInformation( + ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS, + DenseMap &DivInfo, + DenseMap &Multiples, ScalarEvolution &SE) { + // If we have LHS == 0, check if LHS is computing a property of some unknown + // SCEV %v which we can rewrite %v to express explicitly. + if (Predicate != CmpInst::ICMP_EQ || !match(RHS, m_scev_Zero())) + return false; + // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to + // explicitly express that. + const SCEVUnknown *URemLHS = nullptr; + const SCEV *URemRHS = nullptr; + if (!match(LHS, m_scev_URem(m_SCEVUnknown(URemLHS), m_SCEV(URemRHS), SE))) + return false; + + const SCEV *Multiple = + SE.getMulExpr(SE.getUDivExpr(URemLHS, URemRHS), URemRHS); + DivInfo[URemLHS] = Multiple; + if (auto *C = dyn_cast(URemRHS)) + Multiples[URemLHS] = C->getAPInt(); + return true; +} + +// Check if the condition is a divisibility guard (A % B == 0). +static bool isDivisibilityGuard(const SCEV *LHS, const SCEV *RHS, + ScalarEvolution &SE) { + const SCEV *X, *Y; + return match(LHS, m_scev_URem(m_SCEV(X), m_SCEV(Y), SE)) && RHS->isZero(); +} + +// Apply divisibility by \p Divisor on MinMaxExpr with constant values, +// recursively. This is done by aligning up/down the constant value to the +// Divisor. +static const SCEV *applyDivisibilityOnMinMaxExpr(const SCEV *MinMaxExpr, + APInt Divisor, + ScalarEvolution &SE) { + // Return true if \p Expr is a MinMax SCEV expression with a non-negative + // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS + // the non-constant operand and in \p LHS the constant operand. + auto IsMinMaxSCEVWithNonNegativeConstant = + [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS, + const SCEV *&RHS) { + if (auto *MinMax = dyn_cast(Expr)) { + if (MinMax->getNumOperands() != 2) + return false; + if (auto *C = dyn_cast(MinMax->getOperand(0))) { + if (C->getAPInt().isNegative()) + return false; + SCTy = MinMax->getSCEVType(); + LHS = MinMax->getOperand(0); + RHS = MinMax->getOperand(1); + return true; + } + } + return false; + }; + + const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr; + SCEVTypes SCTy; + if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS, + MinMaxRHS)) + return MinMaxExpr; + auto IsMin = isa(MinMaxExpr) || isa(MinMaxExpr); + assert(SE.isKnownNonNegative(MinMaxLHS) && "Expected non-negative operand!"); + auto *DivisibleExpr = + IsMin ? getPreviousSCEVDivisibleByDivisor(MinMaxLHS, Divisor, SE) + : getNextSCEVDivisibleByDivisor(MinMaxLHS, Divisor, SE); + SmallVector Ops = { + applyDivisibilityOnMinMaxExpr(MinMaxRHS, Divisor, SE), DivisibleExpr}; + return SE.getMinMaxExpr(SCTy, Ops); +} + void ScalarEvolution::LoopGuards::collectFromBlock( ScalarEvolution &SE, ScalarEvolution::LoopGuards &Guards, const BasicBlock *Block, const BasicBlock *Pred, @@ -15465,19 +15537,13 @@ void ScalarEvolution::LoopGuards::collectFromBlock( SmallVector ExprsToRewrite; auto CollectCondition = [&](ICmpInst::Predicate Predicate, const SCEV *LHS, const SCEV *RHS, - DenseMap - &RewriteMap) { + DenseMap &RewriteMap, + const LoopGuards &DivGuards) { // WARNING: It is generally unsound to apply any wrap flags to the proposed // replacement SCEV which isn't directly implied by the structure of that // SCEV. In particular, using contextual facts to imply flags is *NOT* // legal. See the scoping rules for flags in the header to understand why. - // If LHS is a constant, apply information to the other expression. - if (isa(LHS)) { - std::swap(LHS, RHS); - Predicate = CmpInst::getSwappedPredicate(Predicate); - } - // Check for a condition of the form (-C1 + X < C2). InstCombine will // create this form when combining two checks of the form (X u< C2 + C1) and // (X >=u C1). @@ -15510,76 +15576,6 @@ void ScalarEvolution::LoopGuards::collectFromBlock( if (MatchRangeCheckIdiom()) return; - // Return true if \p Expr is a MinMax SCEV expression with a non-negative - // constant operand. If so, return in \p SCTy the SCEV type and in \p RHS - // the non-constant operand and in \p LHS the constant operand. - auto IsMinMaxSCEVWithNonNegativeConstant = - [&](const SCEV *Expr, SCEVTypes &SCTy, const SCEV *&LHS, - const SCEV *&RHS) { - if (auto *MinMax = dyn_cast(Expr)) { - if (MinMax->getNumOperands() != 2) - return false; - if (auto *C = dyn_cast(MinMax->getOperand(0))) { - if (C->getAPInt().isNegative()) - return false; - SCTy = MinMax->getSCEVType(); - LHS = MinMax->getOperand(0); - RHS = MinMax->getOperand(1); - return true; - } - } - return false; - }; - - // Apply divisibilty by \p Divisor on MinMaxExpr with constant values, - // recursively. This is done by aligning up/down the constant value to the - // Divisor. - std::function - ApplyDivisibiltyOnMinMaxExpr = [&](const SCEV *MinMaxExpr, - const SCEV *Divisor) { - auto *ConstDivisor = dyn_cast(Divisor); - if (!ConstDivisor) - return MinMaxExpr; - const APInt &DivisorVal = ConstDivisor->getAPInt(); - - const SCEV *MinMaxLHS = nullptr, *MinMaxRHS = nullptr; - SCEVTypes SCTy; - if (!IsMinMaxSCEVWithNonNegativeConstant(MinMaxExpr, SCTy, MinMaxLHS, - MinMaxRHS)) - return MinMaxExpr; - auto IsMin = - isa(MinMaxExpr) || isa(MinMaxExpr); - assert(SE.isKnownNonNegative(MinMaxLHS) && - "Expected non-negative operand!"); - auto *DivisibleExpr = - IsMin - ? getPreviousSCEVDivisibleByDivisor(MinMaxLHS, DivisorVal, SE) - : getNextSCEVDivisibleByDivisor(MinMaxLHS, DivisorVal, SE); - SmallVector Ops = { - ApplyDivisibiltyOnMinMaxExpr(MinMaxRHS, Divisor), DivisibleExpr}; - return SE.getMinMaxExpr(SCTy, Ops); - }; - - // If we have LHS == 0, check if LHS is computing a property of some unknown - // SCEV %v which we can rewrite %v to express explicitly. - if (Predicate == CmpInst::ICMP_EQ && match(RHS, m_scev_Zero())) { - // If LHS is A % B, i.e. A % B == 0, rewrite A to (A /u B) * B to - // explicitly express that. - const SCEVUnknown *URemLHS = nullptr; - const SCEV *URemRHS = nullptr; - if (match(LHS, - m_scev_URem(m_SCEVUnknown(URemLHS), m_SCEV(URemRHS), SE))) { - auto I = RewriteMap.find(URemLHS); - const SCEV *RewrittenLHS = I != RewriteMap.end() ? I->second : URemLHS; - RewrittenLHS = ApplyDivisibiltyOnMinMaxExpr(RewrittenLHS, URemRHS); - const auto *Multiple = - SE.getMulExpr(SE.getUDivExpr(RewrittenLHS, URemRHS), URemRHS); - RewriteMap[URemLHS] = Multiple; - ExprsToRewrite.push_back(URemLHS); - return; - } - } - // Do not apply information for constants or if RHS contains an AddRec. if (isa(LHS) || SE.containsAddRecurrence(RHS)) return; @@ -15609,7 +15605,9 @@ void ScalarEvolution::LoopGuards::collectFromBlock( }; const SCEV *RewrittenLHS = GetMaybeRewritten(LHS); - const APInt &DividesBy = SE.getConstantMultiple(RewrittenLHS); + // Apply divisibility information when computing the constant multiple. + const APInt &DividesBy = + SE.getConstantMultiple(DivGuards.rewrite(RewrittenLHS)); // Collect rewrites for LHS and its transitive operands based on the // condition. @@ -15794,8 +15792,11 @@ void ScalarEvolution::LoopGuards::collectFromBlock( // Now apply the information from the collected conditions to // Guards.RewriteMap. Conditions are processed in reverse order, so the - // earliest conditions is processed first. This ensures the SCEVs with the + // earliest conditions is processed first, except guards with divisibility + // information, which are moved to the back. This ensures the SCEVs with the // shortest dependency chains are constructed first. + SmallVector> + GuardsToProcess; for (auto [Term, EnterIfTrue] : reverse(Terms)) { SmallVector Worklist; SmallPtrSet Visited; @@ -15810,7 +15811,14 @@ void ScalarEvolution::LoopGuards::collectFromBlock( EnterIfTrue ? Cmp->getPredicate() : Cmp->getInversePredicate(); const auto *LHS = SE.getSCEV(Cmp->getOperand(0)); const auto *RHS = SE.getSCEV(Cmp->getOperand(1)); - CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap); + // If LHS is a constant, apply information to the other expression. + // TODO: If LHS is not a constant, check if using CompareSCEVComplexity + // can improve results. + if (isa(LHS)) { + std::swap(LHS, RHS); + Predicate = CmpInst::getSwappedPredicate(Predicate); + } + GuardsToProcess.emplace_back(Predicate, LHS, RHS); continue; } @@ -15823,6 +15831,31 @@ void ScalarEvolution::LoopGuards::collectFromBlock( } } + // Process divisibility guards in reverse order to populate DivGuards early. + DenseMap Multiples; + LoopGuards DivGuards(SE); + for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) { + if (!isDivisibilityGuard(LHS, RHS, SE)) + continue; + collectDivisibilityInformation(Predicate, LHS, RHS, DivGuards.RewriteMap, + Multiples, SE); + } + + for (const auto &[Predicate, LHS, RHS] : GuardsToProcess) + CollectCondition(Predicate, LHS, RHS, Guards.RewriteMap, DivGuards); + + // Apply divisibility information last. This ensures it is applied to the + // outermost expression after other rewrites for the given value. + for (const auto &[K, Divisor] : Multiples) { + const SCEV *DivisorSCEV = SE.getConstant(Divisor); + Guards.RewriteMap[K] = + SE.getMulExpr(SE.getUDivExpr(applyDivisibilityOnMinMaxExpr( + Guards.rewrite(K), Divisor, SE), + DivisorSCEV), + DivisorSCEV); + ExprsToRewrite.push_back(K); + } + // Let the rewriter preserve NUW/NSW flags if the unsigned/signed ranges of // the replacement expressions are contained in the ranges of the replaced // expressions. diff --git a/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll b/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll index b946bbf747083..17c7fe76e410c 100644 --- a/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll +++ b/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll @@ -96,3 +96,201 @@ loop: exit: ret void } + +define i32 @urem_order1(i32 %n) { +; CHECK-LABEL: define i32 @urem_order1( +; CHECK-SAME: i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[UREM:%.*]] = urem i32 [[N]], 3 +; CHECK-NEXT: [[UREM_ZERO:%.*]] = icmp eq i32 [[UREM]], 0 +; CHECK-NEXT: br i1 [[UREM_ZERO]], label %[[PH:.*]], label %[[EXIT:.*]] +; CHECK: [[PH]]: +; CHECK-NEXT: [[N_NON_ZERO:%.*]] = icmp ne i32 [[N]], 0 +; CHECK-NEXT: br i1 [[N_NON_ZERO]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 3 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ 2, %[[PH]] ], [ 3, %[[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %urem = urem i32 %n, 3 + %urem.zero = icmp eq i32 %urem, 0 + br i1 %urem.zero, label %ph, label %exit + +ph: + %n.non.zero = icmp ne i32 %n, 0 + br i1 %n.non.zero, label %loop, label %exit + +loop: + %iv = phi i32 [ 0, %ph ], [ %iv.next, %loop ] + call void @foo() + %iv.next = add i32 %iv, 3 + %ec = icmp eq i32 %iv.next, %n + br i1 %ec, label %exit, label %loop + +exit: + %res = phi i32 [ 1, %entry ], [ 2, %ph ], [ 3, %loop ] + ret i32 %res +} + +define i32 @urem_order2(i32 %n) { +; CHECK-LABEL: define i32 @urem_order2( +; CHECK-SAME: i32 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[N_NON_ZERO:%.*]] = icmp ne i32 [[N]], 0 +; CHECK-NEXT: br i1 [[N_NON_ZERO]], label %[[PH:.*]], label %[[EXIT:.*]] +; CHECK: [[PH]]: +; CHECK-NEXT: [[UREM:%.*]] = urem i32 [[N]], 3 +; CHECK-NEXT: [[UREM_ZERO:%.*]] = icmp eq i32 [[UREM]], 0 +; CHECK-NEXT: br i1 [[UREM_ZERO]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i32 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] +; CHECK-NEXT: call void @foo() +; CHECK-NEXT: [[IV_NEXT]] = add nuw i32 [[IV]], 3 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i32 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT_LOOPEXIT:.*]], label %[[LOOP]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ 2, %[[PH]] ], [ 3, %[[EXIT_LOOPEXIT]] ] +; CHECK-NEXT: ret i32 [[RES]] +; +entry: + %n.non.zero = icmp ne i32 %n, 0 + br i1 %n.non.zero, label %ph, label %exit + +ph: + %urem = urem i32 %n, 3 + %urem.zero = icmp eq i32 %urem, 0 + br i1 %urem.zero, label %loop, label %exit + +loop: + %iv = phi i32 [ 0, %ph ], [ %iv.next, %loop ] + call void @foo() + %iv.next = add i32 %iv, 3 + %ec = icmp eq i32 %iv.next, %n + br i1 %ec, label %exit, label %loop + +exit: + %res = phi i32 [ 1, %entry ], [ 2, %ph ], [ 3, %loop ] + ret i32 %res +} + +define i64 @test_loop_with_div_order_1(i64 %n) { +; CHECK-LABEL: define i64 @test_loop_with_div_order_1( +; CHECK-SAME: i64 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[IS_ZERO:%.*]] = icmp eq i64 [[N]], 0 +; CHECK-NEXT: br i1 [[IS_ZERO]], label %[[EXIT:.*]], label %[[CHECK_BOUNDS:.*]] +; CHECK: [[CHECK_BOUNDS]]: +; CHECK-NEXT: [[N_PLUS_63:%.*]] = add i64 [[N]], 63 +; CHECK-NEXT: [[UPPER_BOUND:%.*]] = lshr i64 [[N_PLUS_63]], 6 +; CHECK-NEXT: [[BOUNDS_CHECK:%.*]] = icmp ult i64 [[N_PLUS_63]], 64 +; CHECK-NEXT: br i1 [[BOUNDS_CHECK]], label %[[EXIT]], label %[[CHECK_PARITY:.*]] +; CHECK: [[CHECK_PARITY]]: +; CHECK-NEXT: [[IS_ODD:%.*]] = and i64 [[N]], 1 +; CHECK-NEXT: [[PARITY_CHECK:%.*]] = icmp eq i64 [[IS_ODD]], 0 +; CHECK-NEXT: br i1 [[PARITY_CHECK]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[DUMMY:%.*]] = load volatile i64, ptr null, align 8 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[UPPER_BOUND]] +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i64 0 +; +entry: + %is_zero = icmp eq i64 %n, 0 + br i1 %is_zero, label %exit, label %check_bounds + +check_bounds: + %n_plus_63 = add i64 %n, 63 + %upper_bound = lshr i64 %n_plus_63, 6 + %bounds_check = icmp ult i64 %n_plus_63, 64 + br i1 %bounds_check, label %exit, label %check_parity + +check_parity: + %is_odd = and i64 %n, 1 + %parity_check = icmp eq i64 %is_odd, 0 + br i1 %parity_check, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv_next, %loop ], [ 0, %check_parity ] + %dummy = load volatile i64, ptr null, align 8 + %iv_next = add i64 %iv, 1 + %exit_cond = icmp ult i64 %iv_next, %upper_bound + br i1 %exit_cond, label %loop, label %exit + +exit: + ret i64 0 +} + +define i64 @test_loop_with_div_order_2(i64 %n) { +; CHECK-LABEL: define i64 @test_loop_with_div_order_2( +; CHECK-SAME: i64 [[N:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[N_PLUS_63:%.*]] = add i64 [[N]], 63 +; CHECK-NEXT: [[UPPER_BOUND:%.*]] = lshr i64 [[N_PLUS_63]], 6 +; CHECK-NEXT: [[BOUNDS_CHECK:%.*]] = icmp ult i64 [[N_PLUS_63]], 64 +; CHECK-NEXT: br i1 [[BOUNDS_CHECK]], label %[[EXIT:.*]], label %[[CHECK_BOUNDS:.*]] +; CHECK: [[CHECK_BOUNDS]]: +; CHECK-NEXT: [[IS_ZERO:%.*]] = icmp eq i64 [[N]], 0 +; CHECK-NEXT: br i1 [[IS_ZERO]], label %[[EXIT]], label %[[CHECK_PARITY:.*]] +; CHECK: [[CHECK_PARITY]]: +; CHECK-NEXT: [[IS_ODD:%.*]] = and i64 [[N]], 1 +; CHECK-NEXT: [[PARITY_CHECK:%.*]] = icmp eq i64 [[IS_ODD]], 0 +; CHECK-NEXT: br i1 [[PARITY_CHECK]], label %[[LOOP_PREHEADER:.*]], label %[[EXIT]] +; CHECK: [[LOOP_PREHEADER]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] +; CHECK-NEXT: [[DUMMY:%.*]] = load volatile i64, ptr null, align 8 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[UPPER_BOUND]] +; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]] +; CHECK: [[EXIT_LOOPEXIT]]: +; CHECK-NEXT: br label %[[EXIT]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret i64 0 +; +entry: + %n_plus_63 = add i64 %n, 63 + %upper_bound = lshr i64 %n_plus_63, 6 + %bounds_check = icmp ult i64 %n_plus_63, 64 + br i1 %bounds_check, label %exit, label %check_bounds + +check_bounds: + %is_zero = icmp eq i64 %n, 0 + br i1 %is_zero, label %exit, label %check_parity + +check_parity: + %is_odd = and i64 %n, 1 + %parity_check = icmp eq i64 %is_odd, 0 + br i1 %parity_check, label %loop, label %exit + +loop: + %iv = phi i64 [ %iv_next, %loop ], [ 0, %check_parity ] + %dummy = load volatile i64, ptr null, align 8 + %iv_next = add i64 %iv, 1 + %exit_cond = icmp ult i64 %iv_next, %upper_bound + br i1 %exit_cond, label %loop, label %exit + +exit: + ret i64 0 +} From d67f605af90c517ada4c15f6e8c4526bc999d1a0 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 2 Sep 2025 21:17:50 +0100 Subject: [PATCH 56/60] [Loads] Add additional test coverage for assumptions. (cherry picked from commit 4efde3c8fddc30b853430fc154cac85cfecc5224) --- ...umption-constant-size-needs-loop-guards.ll | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size-needs-loop-guards.ll diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size-needs-loop-guards.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size-needs-loop-guards.ll new file mode 100644 index 0000000000000..37f0d5418b427 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size-needs-loop-guards.ll @@ -0,0 +1,213 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --filter-out-after "scalar.ph:" --version 5 +; RUN: opt -p loop-vectorize -force-vector-width=2 -S %s | FileCheck %s + +define void @loop_guard_on_assume_needed_to_prove_dereferenceable_ptr_arg_noundef(i64 %x, ptr noalias noundef %A, ptr noalias %B, ptr noalias %C) nofree nosync { +; CHECK-LABEL: define void @loop_guard_on_assume_needed_to_prove_dereferenceable_ptr_arg_noundef( +; CHECK-SAME: i64 [[X:%.*]], ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[X]]) ] +; CHECK-NEXT: [[C_X:%.*]] = icmp uge i64 [[X]], 128 +; CHECK-NEXT: br i1 [[C_X]], label %[[LOOP_HEADER_PREHEADER:.*]], [[EXIT:label %.*]] +; CHECK: [[LOOP_HEADER_PREHEADER]]: +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[TMP2]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4 +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 true, [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %A, i64 4), "dereferenceable"(ptr %A, i64 %x) ] + %c.x = icmp uge i64 %x, 128 + br i1 %c.x, label %loop.header, label %exit + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.b = getelementptr i32, ptr %B, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp eq i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %gep.a = getelementptr i32, ptr %A, i64 %iv + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %C, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 32 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + +define void @loop_guard_on_assume_needed_to_prove_dereferenceable(i64 %x, ptr noalias %A, ptr noalias %B, ptr noalias %C) nofree nosync { +; CHECK-LABEL: define void @loop_guard_on_assume_needed_to_prove_dereferenceable( +; CHECK-SAME: i64 [[X:%.*]], ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "noundef"(ptr [[A]]), "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 [[X]]) ] +; CHECK-NEXT: [[C_X:%.*]] = icmp uge i64 [[X]], 128 +; CHECK-NEXT: br i1 [[C_X]], label %[[LOOP_HEADER_PREHEADER:.*]], [[EXIT:label %.*]] +; CHECK: [[LOOP_HEADER_PREHEADER]]: +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP13]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = icmp eq <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP16]], splat (i1 true) +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0 +; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i32 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1 +; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP18:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP16]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP18]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br i1 true, [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; +entry: + call void @llvm.assume(i1 true) [ "noundef"(ptr %A), "align"(ptr %A, i64 4), "dereferenceable"(ptr %A, i64 %x) ] + %c.x = icmp uge i64 %x, 128 + br i1 %c.x, label %loop.header, label %exit + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.b = getelementptr i32, ptr %B, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp eq i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %gep.a = getelementptr i32, ptr %A, i64 %iv + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %C, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 32 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + +define void @loop_guard_on_trip_count_needed_to_prove_dereferenceable(i32 %x, ptr noalias dereferenceable(128) align 4 %A, ptr noalias %B, ptr noalias %C) { +; CHECK-LABEL: define void @loop_guard_on_trip_count_needed_to_prove_dereferenceable( +; CHECK-SAME: i32 [[X:%.*]], ptr noalias align 4 dereferenceable(128) [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[C_X:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: br i1 [[C_X]], label %[[PH:.*]], [[EXIT:label %.*]] +; CHECK: [[PH]]: +; CHECK-NEXT: [[N:%.*]] = tail call i32 @llvm.smin.i32(i32 [[X]], i32 31) +; CHECK-NEXT: [[N_EXT:%.*]] = zext i32 [[N]] to i64 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N_EXT]], 2 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_EXT]], 2 +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_EXT]], [[N_MOD_VF]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[TMP2]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4 +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[WIDE_LOAD1]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_EXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[CMP_N]], [[EXIT_LOOPEXIT:label %.*]], label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; +entry: + %c.x = icmp sgt i32 %x, 0 + br i1 %c.x, label %ph, label %exit + +ph: + %n = tail call i32 @llvm.smin.i32(i32 %x, i32 31) + %n.ext = zext i32 %n to i64 + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %ph ], [ %iv.next, %loop.latch ] + %gep.b = getelementptr i32, ptr %B, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp eq i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %gep.a = getelementptr i32, ptr %A, i64 %iv + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %C, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, %n.ext + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + + + +declare i32 @llvm.smin.i32(i32, i32) From ee667148bf412d20386908347397cea4be7c6da4 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 3 Oct 2025 14:44:58 +0100 Subject: [PATCH 57/60] [LAA] Check if Ptr can be freed between Assume and CtxI. (#161725) When using information from dereferenceable assumptions, we need to make sure that the memory is not freed between the assume and the specified context instruction. Instead of just checking canBeFreed, check if there any calls that may free between the assume and the context instruction. This patch introduces a willNotFreeBetween to check for calls that may free between an assume and a context instructions, to also be used in https://github.com/llvm/llvm-project/pull/161255. PR: https://github.com/llvm/llvm-project/pull/161725 (cherry picked from commit 7ceef762c8c914270cfa49ad5794227b270c8223) --- llvm/include/llvm/Analysis/ValueTracking.h | 6 + llvm/lib/Analysis/LoopAccessAnalysis.cpp | 27 ++-- llvm/lib/Analysis/ValueTracking.cpp | 26 ++++ .../early-exit-runtime-checks.ll | 126 ++++++++++++++++++ 4 files changed, 177 insertions(+), 8 deletions(-) diff --git a/llvm/include/llvm/Analysis/ValueTracking.h b/llvm/include/llvm/Analysis/ValueTracking.h index 02990a3cb44f7..3e1e718114008 100644 --- a/llvm/include/llvm/Analysis/ValueTracking.h +++ b/llvm/include/llvm/Analysis/ValueTracking.h @@ -618,6 +618,12 @@ LLVM_ABI bool isValidAssumeForContext(const Instruction *I, const DominatorTree *DT = nullptr, bool AllowEphemerals = false); +/// Returns true, if no instruction between \p Assume and \p CtxI may free +/// memory and the function is marked as NoSync. The latter ensures the current +/// function cannot arrange for another thread to free on its behalf. +LLVM_ABI bool willNotFreeBetween(const Instruction *Assume, + const Instruction *CtxI); + enum class OverflowResult { /// Always overflows in the direction of signed/unsigned min value. AlwaysOverflowsLow, diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 1b8ef1b96ad94..02a6a66362493 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -233,14 +233,25 @@ static bool evaluatePtrAddRecAtMaxBTCWillNotWrap( const SCEV *DerefBytesSCEV = SE.getConstant(WiderTy, DerefBytes); // Check if we have a suitable dereferencable assumption we can use. - if (!StartPtrV->canBeFreed()) { - RetainedKnowledge DerefRK = getKnowledgeValidInContext( - StartPtrV, {Attribute::Dereferenceable}, *AC, - L->getLoopPredecessor()->getTerminator(), DT); - if (DerefRK) { - DerefBytesSCEV = - SE.getUMaxExpr(DerefBytesSCEV, SE.getSCEV(DerefRK.IRArgValue)); - } + Instruction *CtxI = &*L->getHeader()->getFirstNonPHIIt(); + if (BasicBlock *LoopPred = L->getLoopPredecessor()) { + if (isa(LoopPred->getTerminator())) + CtxI = LoopPred->getTerminator(); + } + RetainedKnowledge DerefRK; + getKnowledgeForValue(StartPtrV, {Attribute::Dereferenceable}, *AC, + [&](RetainedKnowledge RK, Instruction *Assume, auto) { + if (!isValidAssumeForContext(Assume, CtxI, DT)) + return false; + if (StartPtrV->canBeFreed() && + !willNotFreeBetween(Assume, CtxI)) + return false; + DerefRK = std::max(DerefRK, RK); + return true; + }); + if (DerefRK) { + DerefBytesSCEV = + SE.getUMaxExpr(DerefBytesSCEV, SE.getSCEV(DerefRK.IRArgValue)); } if (DerefBytesSCEV->isZero()) diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index 21f844c4d2f45..161ad2adba85e 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -90,6 +90,9 @@ using namespace llvm::PatternMatch; static cl::opt DomConditionsMaxUses("dom-conditions-max-uses", cl::Hidden, cl::init(20)); +/// Maximum number of instructions to check between assume and context +/// instruction. +static constexpr unsigned MaxInstrsToCheckForFree = 16; /// Returns the bitwidth of the given scalar or pointer type. For vector types, /// returns the element type's bitwidth. @@ -552,6 +555,29 @@ bool llvm::isValidAssumeForContext(const Instruction *Inv, return false; } +bool llvm::willNotFreeBetween(const Instruction *Assume, + const Instruction *CtxI) { + if (CtxI->getParent() != Assume->getParent() || !Assume->comesBefore(CtxI)) + return false; + // Make sure the current function cannot arrange for another thread to free on + // its behalf. + if (!CtxI->getFunction()->hasNoSync()) + return false; + + // Check if there are any calls between the assume and CtxI that may + // free memory. + for (const auto &[Idx, I] : + enumerate(make_range(Assume->getIterator(), CtxI->getIterator()))) { + // Limit number of instructions to walk. + if (Idx > MaxInstrsToCheckForFree) + return false; + if (const auto *CB = dyn_cast(&I)) + if (!CB->hasFnAttr(Attribute::NoFree)) + return false; + } + return true; +} + // TODO: cmpExcludesZero misses many cases where `RHS` is non-constant but // we still have enough information about `RHS` to conclude non-zero. For // example Pred=EQ, RHS=isKnownNonZero. cmpExcludesZero is called in loops diff --git a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll index 207a44d5d08d4..b7f2ae0529d32 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/early-exit-runtime-checks.ll @@ -624,3 +624,129 @@ e.1: e.2: ret void } + +define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption_nofree_via_context(ptr %A, ptr %B) nosync { +; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption_nofree_via_context' +; CHECK-NEXT: loop.header: +; CHECK-NEXT: Memory dependences are safe with run-time checks +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Check 0: +; CHECK-NEXT: Comparing group GRP0: +; CHECK-NEXT: %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv +; CHECK-NEXT: Against group GRP1: +; CHECK-NEXT: %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv +; CHECK-NEXT: Grouped accesses: +; CHECK-NEXT: Group GRP0: +; CHECK-NEXT: (Low: %B High: (2000 + %B)) +; CHECK-NEXT: Member: {%B,+,4}<%loop.header> +; CHECK-NEXT: Group GRP1: +; CHECK-NEXT: (Low: %A High: (2000 + %A)) +; CHECK-NEXT: Member: {%A,+,4}<%loop.header> +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 2000) ] + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %B, i64 2000) ] + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] + %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv + %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv + %l = load i32, ptr %gep.A, align 4 + store i32 0, ptr %gep.B, align 4 + %cntable.c.1 = icmp ult i64 %iv, 1000 + %iv.next = add nuw nsw i64 %iv, 1 + br i1 %cntable.c.1, label %b2, label %e.1 + +b2: + %uncntable.c.0 = icmp eq i32 %l, 0 + br i1 %uncntable.c.0, label %e.2, label %b3 + +b3: + %cntable.c.2 = icmp eq i64 %iv.next, 500 + br i1 %cntable.c.2, label %cleanup4, label %latch + +latch: + br label %loop.header + +cleanup4: + ret void + +e.1: + ret void + +e.2: + ret void +} + +define void @all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption_missing_nofree_multiple_predecessors(ptr %A, ptr %B, i1 %c) nosync { +; CHECK-LABEL: 'all_exits_dominate_latch_countable_exits_at_most_500_iterations_known_deref_via_assumption_missing_nofree_multiple_predecessors' +; CHECK-NEXT: loop.header: +; CHECK-NEXT: Memory dependences are safe with run-time checks +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Check 0: +; CHECK-NEXT: Comparing group GRP0: +; CHECK-NEXT: %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv +; CHECK-NEXT: Against group GRP1: +; CHECK-NEXT: %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv +; CHECK-NEXT: Grouped accesses: +; CHECK-NEXT: Group GRP0: +; CHECK-NEXT: (Low: %B High: inttoptr (i64 -1 to ptr)) +; CHECK-NEXT: Member: {%B,+,4}<%loop.header> +; CHECK-NEXT: Group GRP1: +; CHECK-NEXT: (Low: %A High: inttoptr (i64 -1 to ptr)) +; CHECK-NEXT: Member: {%A,+,4}<%loop.header> +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %A, i64 2000) ] + call void @llvm.assume(i1 true) [ "dereferenceable"(ptr %B, i64 2000) ] + br i1 %c, label %then, label %else + +then: + br label %loop.header + +else: + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %then ], [ 0, %else ], [ %iv.next, %latch ] + %gep.A = getelementptr inbounds i32, ptr %A, i64 %iv + %gep.B = getelementptr inbounds i32, ptr %B, i64 %iv + %l = load i32, ptr %gep.A, align 4 + store i32 0, ptr %gep.B, align 4 + %cntable.c.1 = icmp ult i64 %iv, 1000 + %iv.next = add nuw nsw i64 %iv, 1 + br i1 %cntable.c.1, label %b2, label %e.1 + +b2: + %uncntable.c.0 = icmp eq i32 %l, 0 + br i1 %uncntable.c.0, label %e.2, label %b3 + +b3: + %cntable.c.2 = icmp eq i64 %iv.next, 500 + br i1 %cntable.c.2, label %cleanup4, label %latch + +latch: + br label %loop.header + +cleanup4: + ret void + +e.1: + ret void + +e.2: + ret void +} From 5c9f0df0a8623ac37972478f5a62a23ab6164e2f Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Fri, 5 Sep 2025 15:51:54 +0100 Subject: [PATCH 58/60] [Clang] Allow non-constant sizes for __builtin_assume_dereferenceable. (#156929) Update Clang's __builtin_assume_dereferenceable to support non-constant lengths. The corresponding assume bundle has been updated to support non-constant sizes in cad62df49a7. The current docs for the builtin don't mention the constant requirement for the size argument, so don't need to be updated: https://clang.llvm.org/docs/LanguageExtensions.html#builtin-assume-dereferenceable A number of patches landed recently to make the optimizer make better use of the dereferenceable assumptions, and once https://github.com/llvm/llvm-project/pull/156730 lands, it can be used to vectorize some early-exit loops, for example std::find with std::vector::iterator: https://godbolt.org/z/qo58PKG37 ``` #include #include #include auto find(std::vector::iterator first, short s, unsigned size) { auto Addr = __builtin_assume_aligned(std::to_address(first), 2); __builtin_assume_dereferenceable(std::to_address(first), size * sizeof(short)); return std::find(first, first + size, s); } ``` PR: https://github.com/llvm/llvm-project/pull/156929 (cherry picked from commit c8d065bf914d7c8feb06aa7978fe43b2a800b17f) --- clang/docs/ReleaseNotes.rst | 2 + clang/include/clang/Basic/Builtins.td | 2 +- .../CodeGen/builtin-assume-dereferenceable.c | 59 +++++++++++++++++++ .../builtin-assume-dereferenceable.cpp | 9 ++- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index e5d496737d462..98034cfcd8413 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -389,6 +389,8 @@ Non-comprehensive list of changes in this release this build without optimizations (i.e. use `-O0` or use the `optnone` function attribute) or use the `fno-sanitize-merge=` flag in optimized builds. +- ``__builtin_assume_dereferenceable`` now accepts non-constant size operands. + New Compiler Flags ------------------ - New option ``-fno-sanitize-debug-trap-reasons`` added to disable emitting trap reasons into the debug info when compiling with trapping UBSan (e.g. ``-fsanitize-trap=undefined``). diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td index 0b3ba89dfceb3..f63fc0f43ea7e 100644 --- a/clang/include/clang/Basic/Builtins.td +++ b/clang/include/clang/Basic/Builtins.td @@ -854,7 +854,7 @@ def BuiltinAssumeAligned : Builtin { def BuiltinAssumeDereferenceable : Builtin { let Spellings = ["__builtin_assume_dereferenceable"]; let Attributes = [NoThrow, Const]; - let Prototype = "void(void const*, _Constant size_t)"; + let Prototype = "void(void const*, size_t)"; } def BuiltinFree : Builtin { diff --git a/clang/test/CodeGen/builtin-assume-dereferenceable.c b/clang/test/CodeGen/builtin-assume-dereferenceable.c index cadffd4a84c26..0dc4ba089ee3a 100644 --- a/clang/test/CodeGen/builtin-assume-dereferenceable.c +++ b/clang/test/CodeGen/builtin-assume-dereferenceable.c @@ -32,3 +32,62 @@ int test2(int *a) { __builtin_assume_dereferenceable(a, 32ull); return a[0]; } + +// CHECK-LABEL: @test3( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: [[N_ADDR:%.*]] = alloca i32, align 4 +// CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8 +// CHECK-NEXT: store i32 [[N:%.*]], ptr [[N_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[N_ADDR]], align 4 +// CHECK-NEXT: [[CONV:%.*]] = sext i32 [[TMP1]] to i64 +// CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[TMP0]], i64 [[CONV]]) ] +// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 8 +// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +// CHECK-NEXT: ret i32 [[TMP3]] +// +int test3(int *a, int n) { + __builtin_assume_dereferenceable(a, n); + return a[0]; +} + +// CHECK-LABEL: @test4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: [[N_ADDR:%.*]] = alloca i64, align 8 +// CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8 +// CHECK-NEXT: store i64 [[N:%.*]], ptr [[N_ADDR]], align 8 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load i64, ptr [[N_ADDR]], align 8 +// CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[TMP0]], i64 [[TMP1]]) ] +// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 8 +// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +// CHECK-NEXT: ret i32 [[TMP3]] +// +int test4(int *a, unsigned long long n) { + __builtin_assume_dereferenceable(a, n); + return a[0]; +} + +// CHECK-LABEL: @test5( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[A_ADDR:%.*]] = alloca ptr, align 8 +// CHECK-NEXT: [[N_ADDR:%.*]] = alloca float, align 4 +// CHECK-NEXT: store ptr [[A:%.*]], ptr [[A_ADDR]], align 8 +// CHECK-NEXT: store float [[N:%.*]], ptr [[N_ADDR]], align 4 +// CHECK-NEXT: [[TMP0:%.*]] = load ptr, ptr [[A_ADDR]], align 8 +// CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[N_ADDR]], align 4 +// CHECK-NEXT: [[CONV:%.*]] = fptoui float [[TMP1]] to i64 +// CHECK-NEXT: call void @llvm.assume(i1 true) [ "dereferenceable"(ptr [[TMP0]], i64 [[CONV]]) ] +// CHECK-NEXT: [[TMP2:%.*]] = load ptr, ptr [[A_ADDR]], align 8 +// CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 0 +// CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +// CHECK-NEXT: ret i32 [[TMP3]] +// +int test5(int *a, float n) { + __builtin_assume_dereferenceable(a, n); + return a[0]; +} diff --git a/clang/test/SemaCXX/builtin-assume-dereferenceable.cpp b/clang/test/SemaCXX/builtin-assume-dereferenceable.cpp index b79b7c059567e..6380a9a7debf7 100644 --- a/clang/test/SemaCXX/builtin-assume-dereferenceable.cpp +++ b/clang/test/SemaCXX/builtin-assume-dereferenceable.cpp @@ -18,12 +18,12 @@ int test3(int *a) { } int test4(int *a, unsigned size) { - a = __builtin_assume_dereferenceable(a, size); // expected-error {{argument to '__builtin_assume_dereferenceable' must be a constant integer}} + __builtin_assume_dereferenceable(a, size); return a[0]; } int test5(int *a, unsigned long long size) { - a = __builtin_assume_dereferenceable(a, size); // expected-error {{argument to '__builtin_assume_dereferenceable' must be a constant integer}} + __builtin_assume_dereferenceable(a, size); return a[0]; } @@ -53,3 +53,8 @@ constexpr void *l = __builtin_assume_dereferenceable(p, 4); // expected-error {{ void *foo() { return l; } + +int test10(int *a) { + __builtin_assume_dereferenceable(a, a); // expected-error {{cannot initialize a parameter of type 'unsigned long' with an lvalue of type 'int *'}} + return a[0]; +} From c83eacd8f2d6d9f72a345b4c1eaf1a8186109fd2 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 7 Oct 2025 10:15:25 +0100 Subject: [PATCH 59/60] [Loads] Check if Ptr can be freed between Assume and CtxI. (#161255) When using information from dereferenceable assumptions, we need to make sure that the memory is not freed between the assume and the specified context instruction. Instead of just checking canBeFreed, check if there any calls that may free between the assume and the context instruction. Note that this also adjusts the context instruction to be the terminator in the loop predecessor, if there is one and it is a branch (to avoid things like invoke). PR: https://github.com/llvm/llvm-project/pull/161255 (cherry picked from commit 8b8c59c31f41e0a9dad4ca85fdea0f7249c251ee) --- llvm/lib/Analysis/Loads.cpp | 24 +- ...able-info-from-assumption-constant-size.ll | 321 +++++++++++++++--- .../single-early-exit-deref-assumptions.ll | 39 ++- 3 files changed, 322 insertions(+), 62 deletions(-) diff --git a/llvm/lib/Analysis/Loads.cpp b/llvm/lib/Analysis/Loads.cpp index 0d708f1093c28..ff0a9966551ce 100644 --- a/llvm/lib/Analysis/Loads.cpp +++ b/llvm/lib/Analysis/Loads.cpp @@ -36,17 +36,13 @@ static bool isDereferenceableAndAlignedPointerViaAssumption( function_ref CheckSize, const DataLayout &DL, const Instruction *CtxI, AssumptionCache *AC, const DominatorTree *DT) { - // Dereferenceable information from assumptions is only valid if the value - // cannot be freed between the assumption and use. For now just use the - // information for values that cannot be freed in the function. - // TODO: More precisely check if the pointer can be freed between assumption - // and use. - if (!CtxI || Ptr->canBeFreed()) + if (!CtxI) return false; /// Look through assumes to see if both dereferencability and alignment can /// be proven by an assume if needed. RetainedKnowledge AlignRK; RetainedKnowledge DerefRK; + bool PtrCanBeFreed = Ptr->canBeFreed(); bool IsAligned = Ptr->getPointerAlignment(DL) >= Alignment; return getKnowledgeForValue( Ptr, {Attribute::Dereferenceable, Attribute::Alignment}, *AC, @@ -55,7 +51,11 @@ static bool isDereferenceableAndAlignedPointerViaAssumption( return false; if (RK.AttrKind == Attribute::Alignment) AlignRK = std::max(AlignRK, RK); - if (RK.AttrKind == Attribute::Dereferenceable) + + // Dereferenceable information from assumptions is only valid if the + // value cannot be freed between the assumption and use. + if ((!PtrCanBeFreed || willNotFreeBetween(Assume, CtxI)) && + RK.AttrKind == Attribute::Dereferenceable) DerefRK = std::max(DerefRK, RK); IsAligned |= AlignRK && AlignRK.ArgValue >= Alignment.value(); if (IsAligned && DerefRK && CheckSize(DerefRK)) @@ -390,7 +390,11 @@ bool llvm::isDereferenceableAndAlignedInLoop( } else return false; - Instruction *HeaderFirstNonPHI = &*L->getHeader()->getFirstNonPHIIt(); + Instruction *CtxI = &*L->getHeader()->getFirstNonPHIIt(); + if (BasicBlock *LoopPred = L->getLoopPredecessor()) { + if (isa(LoopPred->getTerminator())) + CtxI = LoopPred->getTerminator(); + } return isDereferenceableAndAlignedPointerViaAssumption( Base, Alignment, [&SE, AccessSizeSCEV, &LoopGuards](const RetainedKnowledge &RK) { @@ -399,9 +403,9 @@ bool llvm::isDereferenceableAndAlignedInLoop( SE.applyLoopGuards(AccessSizeSCEV, *LoopGuards), SE.applyLoopGuards(SE.getSCEV(RK.IRArgValue), *LoopGuards)); }, - DL, HeaderFirstNonPHI, AC, &DT) || + DL, CtxI, AC, &DT) || isDereferenceableAndAlignedPointer(Base, Alignment, AccessSize, DL, - HeaderFirstNonPHI, AC, &DT); + CtxI, AC, &DT); } static bool suppressSpeculativeLoadForSanitizers(const Instruction &CtxI) { diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll index 7a54519c7cdf8..c7c390f8859c1 100644 --- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll +++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll @@ -1587,41 +1587,286 @@ exit: declare ptr @get_ptr() declare void @may_free() -;. -; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} -; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} -; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} -; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} -; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} -; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} -; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} -; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} -; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} -; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} -; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} -; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} -; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} -; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} -; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]} -; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]} -; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]} -; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]} -; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]} -; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]} -; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]} -; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]} -; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]], [[META2]]} -; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META2]], [[META1]]} -; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]} -; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META2]], [[META1]]} -; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]], [[META2]]} -; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]} -; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]} -; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]} -; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]} -; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]} -; CHECK: [[LOOP32]] = distinct !{[[LOOP32]], [[META1]], [[META2]]} -; CHECK: [[LOOP33]] = distinct !{[[LOOP33]], [[META2]], [[META1]]} -; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[META1]], [[META2]]} -; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[META2]], [[META1]]} -;. +define void @deref_assumption_in_header_constant_trip_count_nofree_via_context(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nosync { +; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_nofree_via_context( +; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR2:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD1]], <2 x i32> [[WIDE_LOAD2]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br [[EXIT:label %.*]] +; CHECK: [[SCALAR_PH:.*:]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp sge i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 1000 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + +define void @deref_assumption_in_header_constant_trip_count_may_free(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nosync { +; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_may_free( +; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] +; CHECK-NEXT: call void @may_free() +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0 +; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP17:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP17]] +; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP9:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP8]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x i1> [[TMP4]], i32 1 +; CHECK-NEXT: br i1 [[TMP10]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP18:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP18]] +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP12]], i32 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP14]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP15]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br [[EXIT:label %.*]] +; CHECK: [[SCALAR_PH:.*:]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] + call void @may_free() + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp sge i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 1000 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + +define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_but_missing_nosync(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) { +; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_but_missing_nosync( +; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0 +; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i32 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1 +; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br [[EXIT:label %.*]] +; CHECK: [[SCALAR_PH:.*:]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp sge i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 1000 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} + +define void @deref_assumption_in_header_constant_trip_count_multiple_loop_predecessors(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c, i1 %pre) nosync { +; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_multiple_loop_predecessors( +; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]], i1 [[PRE:%.*]]) #[[ATTR2]] { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] +; CHECK-NEXT: br i1 [[PRE]], label %[[THEN:.*]], label %[[ELSE:.*]] +; CHECK: [[THEN]]: +; CHECK-NEXT: store i32 0, ptr [[A]], align 4 +; CHECK-NEXT: br label %[[LOOP_HEADER_PREHEADER:.*]] +; CHECK: [[ELSE]]: +; CHECK-NEXT: store i32 0, ptr [[B]], align 4 +; CHECK-NEXT: br label %[[LOOP_HEADER_PREHEADER]] +; CHECK: [[LOOP_HEADER_PREHEADER]]: +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0 +; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] +; CHECK: [[PRED_LOAD_IF]]: +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> poison, i32 [[TMP5]], i32 0 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE]] +; CHECK: [[PRED_LOAD_CONTINUE]]: +; CHECK-NEXT: [[TMP7:%.*]] = phi <2 x i32> [ poison, %[[VECTOR_BODY]] ], [ [[TMP6]], %[[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x i1> [[TMP1]], i32 1 +; CHECK-NEXT: br i1 [[TMP8]], label %[[PRED_LOAD_IF1:.*]], label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_IF1]]: +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 +; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1 +; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] +; CHECK: [[PRED_LOAD_CONTINUE2]]: +; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br [[EXIT:label %.*]] +; CHECK: [[SCALAR_PH:.*:]] +; +entry: + call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] + br i1 %pre, label %then, label %else + +then: + store i32 0, ptr %a + br label %loop.header + +else: + store i32 0, ptr %b + br label %loop.header + +loop.header: + %iv = phi i64 [ 0, %then ], [ 0, %else ], [ %iv.next, %loop.latch ] + %gep.a = getelementptr i32, ptr %a, i64 %iv + %gep.b = getelementptr inbounds i32, ptr %b, i64 %iv + %l.b = load i32, ptr %gep.b, align 4 + %c.1 = icmp sge i32 %l.b, 0 + br i1 %c.1, label %loop.latch, label %loop.then + +loop.then: + %l.a = load i32, ptr %gep.a, align 4 + br label %loop.latch + +loop.latch: + %merge = phi i32 [ %l.a, %loop.then ], [ %l.b, %loop.header ] + %gep.c = getelementptr inbounds i32, ptr %c, i64 %iv + store i32 %merge, ptr %gep.c, align 4 + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv.next, 1000 + br i1 %ec, label %exit, label %loop.header + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index b5b8f6ea14cfc..556e5f3ba38be 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -600,24 +600,35 @@ exit: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size_nofree_via_context(ptr noalias %p1, ptr noalias %p2) nosync { ; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size_nofree_via_context( ; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: entry: +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ] -; CHECK-NEXT: br label [[LOOP:%.*]] -; CHECK: loop: -; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX1]] -; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX2]], align 1 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[ARRAYIDX2]], align 1 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX1]] -; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[TMP1]], align 1 -; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] -; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END:%.*]] -; CHECK: loop.inc: -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX1]], 1 -; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024 -; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]] -; CHECK: loop.end: -; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX1]], [[LOOP]] ], [ -1, [[LOOP_INC]] ] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] +; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 +; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]] +; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) +; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024 +; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] +; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[MIDDLE_SPLIT]]: +; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[LOOP_END:.*]] +; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 true) +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], [[TMP7]] +; CHECK-NEXT: br label %[[LOOP_END]] +; CHECK: [[LOOP_END]]: +; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: ret i64 [[RETVAL]] ; entry: From dec6e123124d0753df0ce3151d7d295ba1316758 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 4 Nov 2025 13:58:54 +0000 Subject: [PATCH 60/60] Update tests. --- .../IndVarSimplify/loop-guard-order.ll | 8 +- ...able-info-from-assumption-constant-size.ll | 216 +++++++++++++++--- .../single-early-exit-deref-assumptions.ll | 51 +++-- 3 files changed, 217 insertions(+), 58 deletions(-) diff --git a/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll b/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll index 17c7fe76e410c..69713e006ebb9 100644 --- a/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll +++ b/llvm/test/Transforms/IndVarSimplify/loop-guard-order.ll @@ -208,8 +208,8 @@ define i64 @test_loop_with_div_order_1(i64 %n) { ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] ; CHECK-NEXT: [[DUMMY:%.*]] = load volatile i64, ptr null, align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[UPPER_BOUND]] -; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]] +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[UPPER_BOUND]] +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]] ; CHECK: [[EXIT_LOOPEXIT]]: ; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: @@ -262,8 +262,8 @@ define i64 @test_loop_with_div_order_2(i64 %n) { ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[LOOP_PREHEADER]] ] ; CHECK-NEXT: [[DUMMY:%.*]] = load volatile i64, ptr null, align 8 ; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[IV_NEXT]], [[UPPER_BOUND]] -; CHECK-NEXT: br i1 [[EXITCOND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]] +; CHECK-NEXT: [[EXIT_COND:%.*]] = icmp ult i64 [[IV_NEXT]], [[UPPER_BOUND]] +; CHECK-NEXT: br i1 [[EXIT_COND]], label %[[LOOP]], label %[[EXIT_LOOPEXIT:.*]] ; CHECK: [[EXIT_LOOPEXIT]]: ; CHECK-NEXT: br label %[[EXIT]] ; CHECK: [[EXIT]]: diff --git a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll index c7c390f8859c1..27f956144aabd 100644 --- a/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll +++ b/llvm/test/Transforms/LoopVectorize/dereferenceable-info-from-assumption-constant-size.ll @@ -1590,27 +1590,51 @@ declare void @may_free() define void @deref_assumption_in_header_constant_trip_count_nofree_via_context(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nosync { ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_nofree_via_context( ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR2:[0-9]+]] { -; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i32, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD1]], zeroinitializer -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = getelementptr i32, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <2 x i32>, ptr [[TMP4]], align 4 ; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD1]], <2 x i32> [[WIDE_LOAD2]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] -; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP6]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 -; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP36:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br [[EXIT:label %.*]] -; CHECK: [[SCALAR_PH:.*:]] +; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH1]] +; CHECK: [[SCALAR_PH1]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; CHECK-NEXT: [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4 +; CHECK-NEXT: [[C_1:%.*]] = icmp sge i32 [[L_B]], 0 +; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 4 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[SCALAR_PH]] ], [ [[L_B]], %[[EXIT]] ] +; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]] +; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_C]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP37:![0-9]+]] +; CHECK: [[EXIT1]]: +; CHECK-NEXT: ret void ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] @@ -1643,17 +1667,19 @@ exit: define void @deref_assumption_in_header_constant_trip_count_may_free(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) nosync { ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_may_free( ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR2]] { -; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] ; CHECK-NEXT: call void @may_free() -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP1]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = xor <2 x i1> [[TMP2]], splat (i1 true) ; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x i1> [[TMP4]], i32 0 ; CHECK-NEXT: br i1 [[TMP5]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] ; CHECK: [[PRED_LOAD_IF]]: @@ -1673,16 +1699,38 @@ define void @deref_assumption_in_header_constant_trip_count_may_free(ptr noalias ; CHECK-NEXT: [[TMP13:%.*]] = insertelement <2 x i32> [[TMP9]], i32 [[TMP12]], i32 1 ; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] ; CHECK: [[PRED_LOAD_CONTINUE2]]: -; CHECK-NEXT: [[TMP14:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP4]], <2 x i32> [[TMP14]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP19:%.*]] = phi <2 x i32> [ [[TMP9]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP2]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP19]] ; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] -; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP20]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 -; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP21:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP16]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP38:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br [[EXIT:label %.*]] -; CHECK: [[SCALAR_PH:.*:]] +; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH1]] +; CHECK: [[SCALAR_PH1]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; CHECK-NEXT: [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4 +; CHECK-NEXT: [[C_1:%.*]] = icmp sge i32 [[L_B]], 0 +; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 4 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[SCALAR_PH]] ], [ [[L_B]], %[[EXIT]] ] +; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]] +; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_C]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP39:![0-9]+]] +; CHECK: [[EXIT1]]: +; CHECK-NEXT: ret void ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] @@ -1716,16 +1764,18 @@ exit: define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_but_missing_nosync(ptr noalias noundef %a, ptr noalias %b, ptr noalias %c) { ; CHECK-LABEL: define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_but_missing_nosync( ; CHECK-SAME: ptr noalias noundef [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[C:%.*]]) { -; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: [[ENTRY:.*]]: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[A]], i64 4), "dereferenceable"(ptr [[A]], i64 4000) ] -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP13]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP16]], splat (i1 true) ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0 ; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] ; CHECK: [[PRED_LOAD_IF]]: @@ -1745,16 +1795,38 @@ define void @deref_assumption_in_header_constant_trip_count_nofree_via_context_b ; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1 ; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] ; CHECK: [[PRED_LOAD_CONTINUE2]]: -; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP16]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP18]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] -; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 -; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP40:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br [[EXIT:label %.*]] -; CHECK: [[SCALAR_PH:.*:]] +; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH1]] +; CHECK: [[SCALAR_PH1]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[ENTRY]] ] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ], [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ] +; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; CHECK-NEXT: [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4 +; CHECK-NEXT: [[C_1:%.*]] = icmp sge i32 [[L_B]], 0 +; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 4 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[SCALAR_PH]] ], [ [[L_B]], %[[EXIT]] ] +; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]] +; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_C]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP41:![0-9]+]] +; CHECK: [[EXIT1]]: +; CHECK-NEXT: ret void ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] @@ -1797,14 +1869,16 @@ define void @deref_assumption_in_header_constant_trip_count_multiple_loop_predec ; CHECK-NEXT: store i32 0, ptr [[B]], align 4 ; CHECK-NEXT: br label %[[LOOP_HEADER_PREHEADER]] ; CHECK: [[LOOP_HEADER_PREHEADER]]: -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 false, label %[[SCALAR_PH1:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[PRED_LOAD_CONTINUE2:.*]] ] ; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = icmp slt <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <2 x i32>, ptr [[TMP13]], align 4 +; CHECK-NEXT: [[TMP16:%.*]] = icmp sge <2 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i1> [[TMP16]], splat (i1 true) ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x i1> [[TMP1]], i32 0 ; CHECK-NEXT: br i1 [[TMP2]], label %[[PRED_LOAD_IF:.*]], label %[[PRED_LOAD_CONTINUE:.*]] ; CHECK: [[PRED_LOAD_IF]]: @@ -1824,16 +1898,38 @@ define void @deref_assumption_in_header_constant_trip_count_multiple_loop_predec ; CHECK-NEXT: [[TMP12:%.*]] = insertelement <2 x i32> [[TMP7]], i32 [[TMP11]], i32 1 ; CHECK-NEXT: br label %[[PRED_LOAD_CONTINUE2]] ; CHECK: [[PRED_LOAD_CONTINUE2]]: -; CHECK-NEXT: [[TMP13:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP1]], <2 x i32> [[TMP13]], <2 x i32> [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <2 x i32> [ [[TMP7]], %[[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], %[[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <2 x i1> [[TMP16]], <2 x i32> [[WIDE_LOAD]], <2 x i32> [[TMP18]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] -; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store <2 x i32> [[PREDPHI]], ptr [[TMP17]], align 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 2 ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1000 -; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP23:![0-9]+]] +; CHECK-NEXT: br i1 [[TMP15]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP42:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br [[EXIT:label %.*]] -; CHECK: [[SCALAR_PH:.*:]] +; CHECK-NEXT: br i1 true, label %[[EXIT1:.*]], label %[[SCALAR_PH1]] +; CHECK: [[SCALAR_PH1]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1000, %[[MIDDLE_BLOCK]] ], [ 0, %[[LOOP_HEADER_PREHEADER]] ] +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], %[[LOOP_LATCH:.*]] ], [ [[BC_RESUME_VAL]], %[[SCALAR_PH1]] ] +; CHECK-NEXT: [[GEP_A:%.*]] = getelementptr i32, ptr [[A]], i64 [[IV]] +; CHECK-NEXT: [[GEP_B:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] +; CHECK-NEXT: [[L_B:%.*]] = load i32, ptr [[GEP_B]], align 4 +; CHECK-NEXT: [[C_1:%.*]] = icmp sge i32 [[L_B]], 0 +; CHECK-NEXT: br i1 [[C_1]], label %[[LOOP_LATCH]], label %[[SCALAR_PH:.*]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[L_A:%.*]] = load i32, ptr [[GEP_A]], align 4 +; CHECK-NEXT: br label %[[LOOP_LATCH]] +; CHECK: [[LOOP_LATCH]]: +; CHECK-NEXT: [[MERGE:%.*]] = phi i32 [ [[L_A]], %[[SCALAR_PH]] ], [ [[L_B]], %[[EXIT]] ] +; CHECK-NEXT: [[GEP_C:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[IV]] +; CHECK-NEXT: store i32 [[MERGE]], ptr [[GEP_C]], align 4 +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV_NEXT]], 1000 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT1]], label %[[EXIT]], !llvm.loop [[LOOP43:![0-9]+]] +; CHECK: [[EXIT1]]: +; CHECK-NEXT: ret void ; entry: call void @llvm.assume(i1 true) [ "align"(ptr %a, i64 4), "dereferenceable"(ptr %a, i64 4000) ] @@ -1870,3 +1966,49 @@ loop.latch: exit: ret void } +;. +; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1} +; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"} +; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]} +; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} +; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META2]], [[META1]]} +; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} +; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META2]], [[META1]]} +; CHECK: [[LOOP8]] = distinct !{[[LOOP8]], [[META1]], [[META2]]} +; CHECK: [[LOOP9]] = distinct !{[[LOOP9]], [[META2]], [[META1]]} +; CHECK: [[LOOP10]] = distinct !{[[LOOP10]], [[META1]], [[META2]]} +; CHECK: [[LOOP11]] = distinct !{[[LOOP11]], [[META2]], [[META1]]} +; CHECK: [[LOOP12]] = distinct !{[[LOOP12]], [[META1]], [[META2]]} +; CHECK: [[LOOP13]] = distinct !{[[LOOP13]], [[META2]], [[META1]]} +; CHECK: [[LOOP14]] = distinct !{[[LOOP14]], [[META1]], [[META2]]} +; CHECK: [[LOOP15]] = distinct !{[[LOOP15]], [[META2]], [[META1]]} +; CHECK: [[LOOP16]] = distinct !{[[LOOP16]], [[META1]], [[META2]]} +; CHECK: [[LOOP17]] = distinct !{[[LOOP17]], [[META2]], [[META1]]} +; CHECK: [[LOOP18]] = distinct !{[[LOOP18]], [[META1]], [[META2]]} +; CHECK: [[LOOP19]] = distinct !{[[LOOP19]], [[META2]], [[META1]]} +; CHECK: [[LOOP20]] = distinct !{[[LOOP20]], [[META1]], [[META2]]} +; CHECK: [[LOOP21]] = distinct !{[[LOOP21]], [[META2]], [[META1]]} +; CHECK: [[LOOP22]] = distinct !{[[LOOP22]], [[META1]], [[META2]]} +; CHECK: [[LOOP23]] = distinct !{[[LOOP23]], [[META2]], [[META1]]} +; CHECK: [[LOOP24]] = distinct !{[[LOOP24]], [[META1]], [[META2]]} +; CHECK: [[LOOP25]] = distinct !{[[LOOP25]], [[META2]], [[META1]]} +; CHECK: [[LOOP26]] = distinct !{[[LOOP26]], [[META1]], [[META2]]} +; CHECK: [[LOOP27]] = distinct !{[[LOOP27]], [[META2]], [[META1]]} +; CHECK: [[LOOP28]] = distinct !{[[LOOP28]], [[META1]], [[META2]]} +; CHECK: [[LOOP29]] = distinct !{[[LOOP29]], [[META2]], [[META1]]} +; CHECK: [[LOOP30]] = distinct !{[[LOOP30]], [[META1]], [[META2]]} +; CHECK: [[LOOP31]] = distinct !{[[LOOP31]], [[META2]], [[META1]]} +; CHECK: [[LOOP32]] = distinct !{[[LOOP32]], [[META1]], [[META2]]} +; CHECK: [[LOOP33]] = distinct !{[[LOOP33]], [[META2]], [[META1]]} +; CHECK: [[LOOP34]] = distinct !{[[LOOP34]], [[META1]], [[META2]]} +; CHECK: [[LOOP35]] = distinct !{[[LOOP35]], [[META2]], [[META1]]} +; CHECK: [[LOOP36]] = distinct !{[[LOOP36]], [[META1]], [[META2]]} +; CHECK: [[LOOP37]] = distinct !{[[LOOP37]], [[META2]], [[META1]]} +; CHECK: [[LOOP38]] = distinct !{[[LOOP38]], [[META1]], [[META2]]} +; CHECK: [[LOOP39]] = distinct !{[[LOOP39]], [[META2]], [[META1]]} +; CHECK: [[LOOP40]] = distinct !{[[LOOP40]], [[META1]], [[META2]]} +; CHECK: [[LOOP41]] = distinct !{[[LOOP41]], [[META2]], [[META1]]} +; CHECK: [[LOOP42]] = distinct !{[[LOOP42]], [[META1]], [[META2]]} +; CHECK: [[LOOP43]] = distinct !{[[LOOP43]], [[META2]], [[META1]]} +;. diff --git a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll index 556e5f3ba38be..d949c77bb7ece 100644 --- a/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll +++ b/llvm/test/Transforms/LoopVectorize/single-early-exit-deref-assumptions.ll @@ -600,35 +600,52 @@ exit: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size_nofree_via_context(ptr noalias %p1, ptr noalias %p2) nosync { ; CHECK-LABEL: define i64 @early_exit_alignment_and_deref_known_via_assumption_with_constant_size_nofree_via_context( ; CHECK-SAME: ptr noalias [[P1:%.*]], ptr noalias [[P2:%.*]]) #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: entry: ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P1]], i64 4), "dereferenceable"(ptr [[P1]], i64 1024) ] ; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr [[P2]], i64 4), "dereferenceable"(ptr [[P2]], i64 1024) ] -; CHECK-NEXT: br label %[[VECTOR_PH:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] -; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX1:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT3:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX1]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[ARRAYIDX2]], align 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[ARRAYIDX2]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i8>, ptr [[TMP9]], align 1 ; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX1]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP1]], align 1 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP1]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load <4 x i8>, ptr [[TMP10]], align 1 ; CHECK-NEXT: [[TMP2:%.*]] = icmp ne <4 x i8> [[WIDE_LOAD]], [[WIDE_LOAD2]] ; CHECK-NEXT: [[INDEX_NEXT3]] = add nuw i64 [[INDEX1]], 4 ; CHECK-NEXT: [[TMP3:%.*]] = freeze <4 x i1> [[TMP2]] ; CHECK-NEXT: [[TMP4:%.*]] = call i1 @llvm.vector.reduce.or.v4i1(<4 x i1> [[TMP3]]) ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT3]], 1024 ; CHECK-NEXT: [[TMP6:%.*]] = or i1 [[TMP4]], [[TMP5]] -; CHECK-NEXT: br i1 [[TMP6]], label %[[MIDDLE_SPLIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] -; CHECK: [[MIDDLE_SPLIT]]: -; CHECK-NEXT: br i1 [[TMP4]], label %[[VECTOR_EARLY_EXIT:.*]], label %[[MIDDLE_BLOCK:.*]] -; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br label %[[LOOP_END:.*]] -; CHECK: [[VECTOR_EARLY_EXIT]]: +; CHECK-NEXT: br i1 [[TMP6]], label [[MIDDLE_SPLIT:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] +; CHECK: middle.split: +; CHECK-NEXT: br i1 [[TMP4]], label [[VECTOR_EARLY_EXIT:%.*]], label [[MIDDLE_BLOCK:%.*]] +; CHECK: middle.block: +; CHECK-NEXT: br i1 true, label [[LOOP_END:%.*]], label [[SCALAR_PH]] +; CHECK: vector.early.exit: ; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.experimental.cttz.elts.i64.v4i1(<4 x i1> [[TMP2]], i1 true) ; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX1]], [[TMP7]] -; CHECK-NEXT: br label %[[LOOP_END]] -; CHECK: [[LOOP_END]]: -; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ -1, %[[MIDDLE_BLOCK]] ], [ [[TMP8]], %[[VECTOR_EARLY_EXIT]] ] +; CHECK-NEXT: br label [[LOOP_END]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: br label [[LOOP:%.*]] +; CHECK: loop: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ [[INDEX_NEXT:%.*]], [[LOOP_INC:%.*]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 [[INDEX]] +; CHECK-NEXT: [[LD1:%.*]] = load i8, ptr [[ARRAYIDX]], align 1 +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i8, ptr [[P2]], i64 [[INDEX]] +; CHECK-NEXT: [[LD2:%.*]] = load i8, ptr [[ARRAYIDX1]], align 1 +; CHECK-NEXT: [[CMP3:%.*]] = icmp eq i8 [[LD1]], [[LD2]] +; CHECK-NEXT: br i1 [[CMP3]], label [[LOOP_INC]], label [[LOOP_END]] +; CHECK: loop.inc: +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[EXITCOND:%.*]] = icmp ne i64 [[INDEX_NEXT]], 1024 +; CHECK-NEXT: br i1 [[EXITCOND]], label [[LOOP]], label [[LOOP_END]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK: loop.end: +; CHECK-NEXT: [[RETVAL:%.*]] = phi i64 [ [[INDEX]], [[LOOP]] ], [ -1, [[LOOP_INC]] ], [ -1, [[MIDDLE_BLOCK]] ], [ [[TMP8]], [[VECTOR_EARLY_EXIT]] ] ; CHECK-NEXT: ret i64 [[RETVAL]] ; entry: