From 863c78cbfc9080af51ff1ec5fd802fc4cb22b53b Mon Sep 17 00:00:00 2001 From: Mikhail Gudim Date: Fri, 3 Oct 2025 08:22:33 -0700 Subject: [PATCH 1/4] [SLPVectorizer] Move size checks (NFC). Add the `analyzeRtStrideCandidate` function. In the future commits we're going to add the capability to widen strided loads to it. So, in this commit, we move the size / type checks into it, since it can possibly change size / type of load. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 91c3d4270b241..7d3d9a7c6435f 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2245,6 +2245,11 @@ class BoUpSLP { Align Alignment, const int64_t Diff, Value *Ptr0, Value *PtrN, StridedPtrInfo &SPtrInfo) const; + bool analyzeRtStrideCandidate(ArrayRef PointerOps, Type *ScalarTy, + Align CommonAlignment, + SmallVectorImpl &SortedIndices, + StridedPtrInfo &SPtrInfo) const; + /// Checks if the given array of loads can be represented as a vectorized, /// scatter or just simple gather. /// \param VL list of loads. @@ -6875,6 +6880,27 @@ bool BoUpSLP::isStridedLoad(ArrayRef PointerOps, Type *ScalarTy, return false; } +bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef PointerOps, + Type *ScalarTy, Align CommonAlignment, + SmallVectorImpl &SortedIndices, + StridedPtrInfo &SPtrInfo) const { + return true; + const unsigned Sz = PointerOps.size(); + // TODO: VecSz may change if we widen the strided load. + unsigned VecSz = Sz; + FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, VecSz); + if (!(Sz > MinProfitableStridedLoads && TTI->isTypeLegal(StridedLoadTy) && + TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))) + return false; + if (const SCEV *Stride = + calculateRtStride(PointerOps, ScalarTy, *DL, *SE, SortedIndices)) { + SPtrInfo.Ty = getWidenedType(ScalarTy, PointerOps.size()); + SPtrInfo.StrideSCEV = Stride; + return true; + } + return false; +} + BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( ArrayRef VL, const Value *VL0, SmallVectorImpl &Order, SmallVectorImpl &PointerOps, StridedPtrInfo &SPtrInfo, @@ -6915,15 +6941,9 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( auto *VecTy = getWidenedType(ScalarTy, Sz); Align CommonAlignment = computeCommonAlignment(VL); if (!IsSorted) { - if (Sz > MinProfitableStridedLoads && TTI->isTypeLegal(VecTy)) { - if (const SCEV *Stride = - calculateRtStride(PointerOps, ScalarTy, *DL, *SE, Order); - Stride && TTI->isLegalStridedLoadStore(VecTy, CommonAlignment)) { - SPtrInfo.Ty = getWidenedType(ScalarTy, PointerOps.size()); - SPtrInfo.StrideSCEV = Stride; - return LoadsState::StridedVectorize; - } - } + if (analyzeRtStrideCandidate(PointerOps, ScalarTy, CommonAlignment, Order, + SPtrInfo)) + return LoadsState::StridedVectorize; if (!TTI->isLegalMaskedGather(VecTy, CommonAlignment) || TTI->forceScalarizeMaskedGather(VecTy, CommonAlignment)) From ba976de9bb544dd8e54fa74e9f3b6773a14da011 Mon Sep 17 00:00:00 2001 From: Mikhail Gudim Date: Fri, 3 Oct 2025 10:11:06 -0700 Subject: [PATCH 2/4] removed wrong `return true`. --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7d3d9a7c6435f..1a7edcc1550e0 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -6884,7 +6884,6 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef PointerOps, Type *ScalarTy, Align CommonAlignment, SmallVectorImpl &SortedIndices, StridedPtrInfo &SPtrInfo) const { - return true; const unsigned Sz = PointerOps.size(); // TODO: VecSz may change if we widen the strided load. unsigned VecSz = Sz; From 509d5f5bbd4ffe55965dc525f648e0772426fddc Mon Sep 17 00:00:00 2001 From: Mikhail Gudim Date: Tue, 7 Oct 2025 07:34:33 -0700 Subject: [PATCH 3/4] addressed review comments. --- .../Transforms/Vectorize/SLPVectorizer.cpp | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 1a7edcc1550e0..963e4994d58de 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2245,6 +2245,21 @@ class BoUpSLP { Align Alignment, const int64_t Diff, Value *Ptr0, Value *PtrN, StridedPtrInfo &SPtrInfo) const; + /// Return true if an array of scalar loads can be replaced with a strided + /// load (with run-time stride). + /// \param PointerOps list of pointer arguments of loads. + /// \param ScalarTy type of loads. + /// \param CommonAlignment common alignement of loads as computed by + /// `computeCommonAlignment`. + /// \param SortedIndicies is a list of indicies computed by this function such + /// that the sequence `PointerOps[SortedIndices[0]], + /// PointerOps[SortedIndicies[1]], ..., PointerOps[SortedIndices[n]]` is + /// ordered by the coefficient of the stride. For example, if PointerOps is + /// `%base + %stride, %base, %base + 2 * stride` the `SortedIndices` will be + /// `[1, 0, 2]`. We follow the convention that if `SortedIndices` has to be + /// `0, 1, 2, 3, ...` we return empty vector for `SortedIndicies`. + /// \param SPtrInfo If the function return `true`, it also sets all the fields + /// of `SPtrInfo` necessary to generate the strided load later. bool analyzeRtStrideCandidate(ArrayRef PointerOps, Type *ScalarTy, Align CommonAlignment, SmallVectorImpl &SortedIndices, @@ -6888,8 +6903,8 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef PointerOps, // TODO: VecSz may change if we widen the strided load. unsigned VecSz = Sz; FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, VecSz); - if (!(Sz > MinProfitableStridedLoads && TTI->isTypeLegal(StridedLoadTy) && - TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment))) + if (Sz <= MinProfitableStridedLoads || !TTI->isTypeLegal(StridedLoadTy) || + !TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment)) return false; if (const SCEV *Stride = calculateRtStride(PointerOps, ScalarTy, *DL, *SE, SortedIndices)) { From 4079b2e7aba976df0bf79f51cf29d17ca0259092 Mon Sep 17 00:00:00 2001 From: Mikhail Gudim Date: Wed, 8 Oct 2025 13:59:16 -0700 Subject: [PATCH 4/4] adderessed review comments --- llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 963e4994d58de..fdd0905f52178 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -6900,9 +6900,7 @@ bool BoUpSLP::analyzeRtStrideCandidate(ArrayRef PointerOps, SmallVectorImpl &SortedIndices, StridedPtrInfo &SPtrInfo) const { const unsigned Sz = PointerOps.size(); - // TODO: VecSz may change if we widen the strided load. - unsigned VecSz = Sz; - FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, VecSz); + FixedVectorType *StridedLoadTy = getWidenedType(ScalarTy, Sz); if (Sz <= MinProfitableStridedLoads || !TTI->isTypeLegal(StridedLoadTy) || !TTI->isLegalStridedLoadStore(StridedLoadTy, CommonAlignment)) return false;