diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index 6c9097c92a86d..f51d8af1b56c9 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -886,8 +886,12 @@ replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, /// /// If necessary this method will version the stride of the pointer according /// to \p PtrToStride and therefore add further predicates to \p PSE. -/// The \p Assume parameter indicates if we are allowed to make additional -/// run-time assumptions. +/// +/// If \p Predicates is non-null, the no-wrap SCEV predicates needed to enable +/// the stride analysis are appended to it rather than added to \p PSE, letting +/// the caller discard them if the result turns out to be unused. (Stride +/// versioning may still add predicates to \p PSE directly.) If \p Predicates is +/// null, no such no-wrap predicates are collected. /// /// Note that the analysis results are defined if-and-only-if the original /// memory access was defined. If that access was dead, or UB, then the @@ -897,7 +901,17 @@ getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, const DominatorTree &DT, const DenseMap &StridesMap = DenseMap(), - bool Assume = false, bool ShouldCheckWrap = true); + bool ShouldCheckWrap = true, + SmallVectorImpl *Predicates = nullptr); + +/// Overload of \ref getPtrStride that adds the no-wrap predicates directly to +/// \p PSE. The \p Assume parameter indicates whether such additional run-time +/// assumptions are allowed. +LLVM_ABI std::optional +getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, + const Loop *Lp, const DominatorTree &DT, + const DenseMap &StridesMap, bool Assume, + bool ShouldCheckWrap = true); /// Returns the distance between the pointers \p PtrA and \p PtrB iff they are /// compatible and it is possible to calculate the distance between them. This diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h index b863a307b143e..b483466979786 100644 --- a/llvm/include/llvm/Analysis/VectorUtils.h +++ b/llvm/include/llvm/Analysis/VectorUtils.h @@ -805,8 +805,12 @@ class InterleavedAccessInfo { delete Group; } - /// Collect all the accesses with a constant stride in program order. + /// Collect all the accesses with a constant stride in program order. Any + /// SCEV predicates needed to compute the strides are appended to \p + /// Predicates rather than being added to PSE, so they can be discarded if no + /// interleave group is formed. void collectConstStrideAccesses( + SmallVectorImpl &Predicates, MapVector &AccessStrideInfo, const DenseMap &Strides); diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 74f0fa8a954da..d6c2bdc73fbeb 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -1022,15 +1022,17 @@ getStrideFromAddRec(const SCEVAddRecExpr *AR, const Loop *Lp, Type *AccessTy, /// Check whether \p AR is a non-wrapping AddRec. If \p Ptr is not nullptr, use /// informating from the IR pointer value to determine no-wrap. -static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, - Value *Ptr, Type *AccessTy, const Loop *L, bool Assume, - const DominatorTree &DT, - std::optional Stride = std::nullopt) { +static bool +isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, Value *Ptr, + Type *AccessTy, const Loop *L, const DominatorTree &DT, + std::optional Stride = std::nullopt, + SmallVectorImpl *Predicates = nullptr) { // FIXME: This should probably only return true for NUW. if (any(AR->getNoWrapFlags(SCEV::NoWrapMask))) return true; - if (Ptr && PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW)) + if (Ptr && isa(PSE.getSCEV(Ptr)) && + PSE.hasNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW)) return true; // An nusw getelementptr that is an AddRec cannot wrap. If it would wrap, @@ -1066,8 +1068,12 @@ static bool isNoWrap(PredicatedScalarEvolution &PSE, const SCEVAddRecExpr *AR, return true; } - if (Ptr && Assume) { - PSE.setNoOverflow(Ptr, SCEVWrapPredicate::IncrementNUSW); + if (Ptr && Predicates) { + ScalarEvolution *SE = PSE.getSE(); + SCEVWrapPredicate::IncrementWrapFlags Flags = SCEVWrapPredicate::clearFlags( + SCEVWrapPredicate::IncrementNUSW, + SCEVWrapPredicate::getImpliedFlags(AR, *SE)); + Predicates->push_back(SE->getWrapPredicate(AR, Flags)); LLVM_DEBUG(dbgs() << "LAA: Pointer may wrap:\n" << "LAA: Pointer: " << *Ptr << "\n" << "LAA: SCEV: " << *AR << "\n" @@ -1295,6 +1301,7 @@ bool AccessAnalysis::createCheckForAccess( /// Check whether all pointers can participate in a runtime bounds check. They /// must either be invariant or non-wrapping affine AddRecs. + SmallVector Predicates; for (auto &P : RTCheckPtrs) { // The bounds for loop-invariant pointer is trivial. if (SE->isLoopInvariant(P.getPointer(), TheLoop)) @@ -1315,9 +1322,12 @@ bool AccessAnalysis::createCheckForAccess( } if (!isNoWrap(PSE, AR, RTCheckPtrs.size() == 1 ? Ptr : nullptr, AccessTy, - TheLoop, Assume, DT)) + TheLoop, DT, /*Stride=*/std::nullopt, + Assume ? &Predicates : nullptr)) return false; } + for (const SCEVPredicate *P : Predicates) + PSE.addPredicate(*P); for (const auto &[PtrExpr, NeedsFreeze] : RTCheckPtrs) { // The id of the dependence set. @@ -1646,11 +1656,10 @@ void AccessAnalysis::buildDependenceSets() { } /// Check whether the access through \p Ptr has a constant stride. -std::optional -llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, - const Loop *Lp, const DominatorTree &DT, - const DenseMap &StridesMap, - bool Assume, bool ShouldCheckWrap) { +std::optional llvm::getPtrStride( + PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, + const DominatorTree &DT, const DenseMap &StridesMap, + bool ShouldCheckWrap, SmallVectorImpl *Predicates) { const SCEV *PtrScev = replaceSymbolicStrideSCEV(PSE, StridesMap, Ptr); if (PSE.getSE()->isLoopInvariant(PtrScev, Lp)) return 0; @@ -1658,8 +1667,10 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, assert(Ptr->getType()->isPointerTy() && "Unexpected non-ptr"); const SCEVAddRecExpr *AR = dyn_cast(PtrScev); - if (Assume && !AR) - AR = PSE.getAsAddRec(Ptr); + if (Predicates && !AR) { + AR = PSE.getSE()->convertSCEVToAddRecWithPredicates(PtrScev, Lp, + *Predicates); + } if (!AR) { LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not an AddRecExpr pointer " << *Ptr @@ -1672,7 +1683,7 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, if (!ShouldCheckWrap || !Stride) return Stride; - if (isNoWrap(PSE, AR, Ptr, AccessTy, Lp, Assume, DT, Stride)) + if (isNoWrap(PSE, AR, Ptr, AccessTy, Lp, DT, Stride, Predicates)) return Stride; LLVM_DEBUG( @@ -1681,6 +1692,21 @@ llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, return std::nullopt; } +/// Check whether the access through \p Ptr has a constant stride. +std::optional +llvm::getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, + const Loop *Lp, const DominatorTree &DT, + const DenseMap &StridesMap, + bool Assume, bool ShouldCheckWrap) { + SmallVector Predicates; + std::optional Stride = + getPtrStride(PSE, AccessTy, Ptr, Lp, DT, StridesMap, ShouldCheckWrap, + Assume ? &Predicates : nullptr); + for (const SCEVPredicate *P : Predicates) + PSE.addPredicate(*P); + return Stride; +} + std::optional llvm::getPointersDiff(Type *ElemTyA, Value *PtrA, Type *ElemTyB, Value *PtrB, const DataLayout &DL, diff --git a/llvm/lib/Analysis/ScalarEvolution.cpp b/llvm/lib/Analysis/ScalarEvolution.cpp index c93838ffdc50d..e1207d2d625bb 100644 --- a/llvm/lib/Analysis/ScalarEvolution.cpp +++ b/llvm/lib/Analysis/ScalarEvolution.cpp @@ -15448,8 +15448,25 @@ bool SCEVUnionPredicate::implies(const SCEVPredicate *N, return this->implies(I, SE); }); - return any_of(Preds, - [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); }); + if (any_of(Preds, + [N, &SE](const SCEVPredicate *I) { return I->implies(N, SE); })) + return true; + + // A wrap predicate may be implied by a wrap predicate in Preds after applying + // equal predicates. + const auto *NWrap = dyn_cast(N); + if (!NWrap) + return false; + const Loop *L = NWrap->getExpr()->getLoop(); + return any_of(Preds, [&](const SCEVPredicate *I) { + const auto *IWrap = dyn_cast(I); + if (!IWrap) + return false; + const auto *RewrittenAR = dyn_cast( + SE.rewriteUsingPredicate(IWrap->getExpr(), L, *this)); + return RewrittenAR && + SE.getWrapPredicate(RewrittenAR, IWrap->getFlags())->implies(N, SE); + }); } void SCEVUnionPredicate::print(raw_ostream &OS, unsigned Depth) const { diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp index ecded79e990f3..28e728bb4c5ac 100644 --- a/llvm/lib/Analysis/VectorUtils.cpp +++ b/llvm/lib/Analysis/VectorUtils.cpp @@ -1308,8 +1308,9 @@ bool InterleavedAccessInfo::isStrided(int Stride) { } void InterleavedAccessInfo::collectConstStrideAccesses( + SmallVectorImpl &Predicates, MapVector &AccessStrideInfo, - const DenseMap &Strides) { + const DenseMap &Strides) { auto &DL = TheLoop->getHeader()->getDataLayout(); // Since it's desired that the load/store instructions be maintained in @@ -1341,7 +1342,7 @@ void InterleavedAccessInfo::collectConstStrideAccesses( // even without the transformation. The wrapping checks are therefore // deferred until after we've formed the interleaved groups. int64_t Stride = getPtrStride(PSE, ElementTy, Ptr, TheLoop, *DT, Strides, - /*Assume=*/true, /*ShouldCheckWrap=*/false) + /*ShouldCheckWrap=*/false, &Predicates) .value_or(0); const SCEV *Scev = replaceSymbolicStrideSCEV(PSE, Strides, Ptr); @@ -1391,9 +1392,14 @@ void InterleavedAccessInfo::analyzeInterleaving( LLVM_DEBUG(dbgs() << "LV: Analyzing interleaved accesses...\n"); const auto &Strides = LAI->getSymbolicStrides(); + // Collect the SCEV predicates needed by the stride analysis below. They are + // only added to PSE if at least one interleave group is formed, so they can + // be discarded otherwise. + SmallVector Predicates; + // Holds all accesses with a constant stride. MapVector AccessStrideInfo; - collectConstStrideAccesses(AccessStrideInfo, Strides); + collectConstStrideAccesses(Predicates, AccessStrideInfo, Strides); if (AccessStrideInfo.empty()) return; @@ -1589,6 +1595,15 @@ void InterleavedAccessInfo::analyzeInterleaving( } // Iteration over A accesses. } // Iteration over B accesses. + // The predicates collected while forming the candidate groups above are + // needed by the wrap-check below (which uses Assume=false and therefore + // relies on the no-wrap assumptions already being available in PSE) and by + // later passes. Commit them now if any candidate group was formed; otherwise + // they are unused and discarded. + if (!LoadGroups.empty() || !StoreGroups.empty()) + for (const SCEVPredicate *P : Predicates) + PSE.addPredicate(*P); + auto InvalidateGroupIfMemberMayWrap = [&](InterleaveGroup *Group, int Index, const char *FirstOrLast) -> bool { diff --git a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll index 63abd4ef70d63..88b16d0f32534 100644 --- a/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll +++ b/llvm/test/Analysis/LoopAccessAnalysis/retry-runtime-checks-after-dependence-analysis.ll @@ -72,27 +72,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_not_inbounds_iv2_st ; CHECK-NEXT: Comparing group GRP0: ; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv ; CHECK-NEXT: Against group GRP1: -; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2 +; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset ; CHECK-NEXT: Check 1: ; CHECK-NEXT: Comparing group GRP0: ; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv ; CHECK-NEXT: Against group GRP2: -; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset +; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2 ; CHECK-NEXT: Check 2: ; CHECK-NEXT: Comparing group GRP1: -; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2 -; CHECK-NEXT: Against group GRP2: ; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset +; CHECK-NEXT: Against group GRP2: +; CHECK-NEXT: %gep.b = getelementptr i8, ptr %b, i64 %iv2 ; CHECK-NEXT: Grouped accesses: ; CHECK-NEXT: Group GRP0: ; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a)) ; CHECK-NEXT: Member: {%a,+,4}<%loop> ; CHECK-NEXT: Group GRP1: -; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b)) -; CHECK-NEXT: Member: {%b,+,5}<%loop> -; CHECK-NEXT: Group GRP2: ; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a)) ; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop> +; CHECK-NEXT: Group GRP2: +; CHECK-NEXT: (Low: %b High: (-1 + (5 * %n) + %b)) +; CHECK-NEXT: Member: {%b,+,5}<%loop> ; CHECK-EMPTY: ; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. ; CHECK-NEXT: SCEV assumptions: @@ -265,27 +265,27 @@ define void @dependency_check_and_runtime_checks_needed_gepb_may_wrap(ptr %a, pt ; CHECK-NEXT: Comparing group GRP0: ; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv ; CHECK-NEXT: Against group GRP1: -; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2 +; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset ; CHECK-NEXT: Check 1: ; CHECK-NEXT: Comparing group GRP0: ; CHECK-NEXT: %gep.a.iv = getelementptr inbounds float, ptr %a, i64 %iv ; CHECK-NEXT: Against group GRP2: -; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset +; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2 ; CHECK-NEXT: Check 2: ; CHECK-NEXT: Comparing group GRP1: -; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2 -; CHECK-NEXT: Against group GRP2: ; CHECK-NEXT: %gep.a.iv.off = getelementptr inbounds float, ptr %a, i64 %iv.offset +; CHECK-NEXT: Against group GRP2: +; CHECK-NEXT: %gep.b = getelementptr float, ptr %b, i64 %iv2 ; CHECK-NEXT: Grouped accesses: ; CHECK-NEXT: Group GRP0: ; CHECK-NEXT: (Low: %a High: ((4 * %n) + %a)) ; CHECK-NEXT: Member: {%a,+,4}<%loop> ; CHECK-NEXT: Group GRP1: -; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b)) -; CHECK-NEXT: Member: {%b,+,8}<%loop> -; CHECK-NEXT: Group GRP2: ; CHECK-NEXT: (Low: ((4 * %offset) + %a) High: ((4 * %offset) + (4 * %n) + %a)) ; CHECK-NEXT: Member: {((4 * %offset) + %a),+,4}<%loop> +; CHECK-NEXT: Group GRP2: +; CHECK-NEXT: (Low: %b High: (-4 + (8 * %n) + %b)) +; CHECK-NEXT: Member: {%b,+,8}<%loop> ; CHECK-EMPTY: ; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. ; CHECK-NEXT: SCEV assumptions: @@ -325,29 +325,29 @@ define void @retry_after_dep_check_with_unknown_offset(ptr %A, i32 %offset) { ; CHECK-NEXT: Run-time memory checks: ; CHECK-NEXT: Check 0: ; CHECK-NEXT: Comparing group GRP0: -; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv +; CHECK-NEXT: ptr %A ; CHECK-NEXT: Against group GRP1: -; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3 +; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv ; CHECK-NEXT: Check 1: ; CHECK-NEXT: Comparing group GRP0: -; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv -; CHECK-NEXT: Against group GRP2: ; CHECK-NEXT: ptr %A +; CHECK-NEXT: Against group GRP2: +; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3 ; CHECK-NEXT: Check 2: ; CHECK-NEXT: Comparing group GRP1: -; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3 +; CHECK-NEXT: %A.100.iv = getelementptr { float, float }, ptr %A.100, i64 %iv ; CHECK-NEXT: Against group GRP2: -; CHECK-NEXT: ptr %A +; CHECK-NEXT: %A.100.iv.offset.3 = getelementptr i8, ptr %A.100, i64 %iv.offset.3 ; CHECK-NEXT: Grouped accesses: ; CHECK-NEXT: Group GRP0: +; CHECK-NEXT: (Low: %A High: (4 + %A)) +; CHECK-NEXT: Member: %A +; CHECK-NEXT: Group GRP1: ; CHECK-NEXT: (Low: (100 + %A) High: (96 + (8 * (zext i32 %offset to i64)) + %A)) ; CHECK-NEXT: Member: {(100 + %A),+,8}<%loop> -; CHECK-NEXT: Group GRP1: +; CHECK-NEXT: Group GRP2: ; CHECK-NEXT: (Low: (100 + (8 * (zext i32 %offset to i64)) + %A) High: (96 + (16 * (zext i32 %offset to i64)) + %A)) ; CHECK-NEXT: Member: {(100 + (8 * (zext i32 %offset to i64)) + %A),+,8}<%loop> -; CHECK-NEXT: Group GRP2: -; CHECK-NEXT: (Low: %A High: (4 + %A)) -; CHECK-NEXT: Member: %A ; CHECK-EMPTY: ; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. ; CHECK-NEXT: SCEV assumptions: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll index 4d21309776b40..1c79629ac1754 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/discarded-interleave-group.ll @@ -3,28 +3,42 @@ target triple = "arm64-apple-macosx" -; FIXME: should vectorize without predicates. define void @urem_lookup(ptr noalias %src, ptr noalias %dst, ptr noalias %tbl, i64 %N) #0 { ; CHECK-LABEL: define void @urem_lookup( ; CHECK-SAME: ptr noalias [[SRC:%.*]], ptr noalias [[DST:%.*]], ptr noalias [[TBL:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: -; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP1:%.*]] = shl nuw i64 [[TMP0]], 2 +; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[N]]) +; CHECK-NEXT: [[TMP2:%.*]] = call @llvm.stepvector.nxv4i64() +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TMP1]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[LOOP]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], %[[LOOP]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP2]], %[[LOOP]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] ; CHECK-NEXT: [[GEP_SRC:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 [[IV]] -; CHECK-NEXT: [[X:%.*]] = load float, ptr [[GEP_SRC]], align 4 -; CHECK-NEXT: [[CLAMPED:%.*]] = urem i64 [[IV]], 4 -; CHECK-NEXT: [[GEP_TBL:%.*]] = getelementptr inbounds float, ptr [[TBL]], i64 [[CLAMPED]] -; CHECK-NEXT: [[T:%.*]] = load float, ptr [[GEP_TBL]], align 4 -; CHECK-NEXT: [[M1:%.*]] = fmul float [[X]], [[T]] -; CHECK-NEXT: [[M2:%.*]] = fadd float [[M1]], [[X]] -; CHECK-NEXT: [[M3:%.*]] = fmul float [[M2]], [[T]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr align 4 [[GEP_SRC]], [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP4:%.*]] = urem [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[TBL]], [[TMP4]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( align 4 [[TMP5]], [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP6:%.*]] = fmul [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_GATHER]] +; CHECK-NEXT: [[TMP7:%.*]] = fadd [[TMP6]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: [[TMP8:%.*]] = fmul [[TMP7]], [[WIDE_MASKED_GATHER]] ; CHECK-NEXT: [[GEP_DST:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 [[IV]] -; CHECK-NEXT: store float [[M3]], ptr [[GEP_DST]], align 4 -; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 -; CHECK-NEXT: [[COND:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[COND]], label %[[EXIT:.*]], label %[[LOOP]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0( [[TMP8]], ptr align 4 [[GEP_DST]], [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[IV]], [[TMP1]] +; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[N]]) +; CHECK-NEXT: [[TMP10:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 +; CHECK-NEXT: [[TMP11:%.*]] = xor i1 [[TMP10]], true +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw [[VEC_IND]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: br i1 [[TMP11]], label %[[EXIT:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[EXIT1:.*]] +; CHECK: [[EXIT1]]: ; CHECK-NEXT: ret void ; entry: