From 6d66db3890a18e3926a49cbfeb28e99c464cfcd5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sat, 13 Apr 2024 08:14:40 +0100 Subject: [PATCH] [SLP] Initial vectorization of non-power-of-2 ops. (#77790) This patch enables vectorization for non-power-of-2 VFs. Initially only VFs where adding 1 makes the VF a power-of-2, i.e. we can still make relatively effective use of the vectors. It relies on the existing target cost-models to return accurate costs for non-power-of-2 vectors. I checked mostly AArch64 and X86 and there the costs seem reasonable for the costs I checked, although I expect there will be a need to refine both the cost-models and lowering to make most effective use of non-power-of-2 SLP vectorization. Note that re-ordering and shuffling is not implemented for nodes requiring padding yet to keep the initial implementation simpler. The feature is guarded by a new flag, off by defaul for now. PR: https://github.com/llvm/llvm-project/pull/77790 --- .../Transforms/Vectorize/SLPVectorizer.cpp | 83 ++++- .../SLPVectorizer/AArch64/vec15-base.ll | 70 ++-- .../SLPVectorizer/AArch64/vec3-base.ll | 247 +++++++++----- .../SLPVectorizer/AArch64/vec3-calls.ll | 3 +- .../AArch64/vec3-reorder-reshuffle.ll | 266 ++++++++++----- .../Transforms/SLPVectorizer/X86/odd_store.ll | 84 +++-- .../Transforms/SLPVectorizer/X86/vec3-base.ll | 35 +- .../SLPVectorizer/X86/vec3-calls.ll | 33 +- .../X86/vec3-gather-some-loads.ll | 74 +++-- .../X86/vec3-reorder-reshuffle.ll | 248 ++++++++------ .../X86/vect_copyable_in_binops.ll | 303 ++++++++++-------- 11 files changed, 947 insertions(+), 499 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 7b5b0efca8d3f..09db50132c892 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -190,6 +190,10 @@ static cl::opt ViewSLPTree("view-slp-tree", cl::Hidden, cl::desc("Display the SLP trees with Graphviz")); +static cl::opt VectorizeNonPowerOf2( + "slp-vectorize-non-power-of-2", cl::init(false), cl::Hidden, + cl::desc("Try to vectorize with non-power-of-2 number of elements.")); + // Limit the number of alias checks. The limit is chosen so that // it has no negative effect on the llvm benchmarks. static const unsigned AliasedCheckLimit = 10; @@ -2829,6 +2833,14 @@ class BoUpSLP { SmallVectorImpl *OpScalars = nullptr, SmallVectorImpl *AltScalars = nullptr) const; + /// Return true if this is a non-power-of-2 node. + bool isNonPowOf2Vec() const { + bool IsNonPowerOf2 = !isPowerOf2_32(Scalars.size()); + assert((!IsNonPowerOf2 || ReuseShuffleIndices.empty()) && + "Reshuffling not supported with non-power-of-2 vectors yet."); + return IsNonPowerOf2; + } + #ifndef NDEBUG /// Debug printer. LLVM_DUMP_METHOD void dump() const { @@ -2994,9 +3006,11 @@ class BoUpSLP { MustGather.insert(VL.begin(), VL.end()); } - if (UserTreeIdx.UserTE) + if (UserTreeIdx.UserTE) { Last->UserTreeIndices.push_back(UserTreeIdx); - + assert((!Last->isNonPowOf2Vec() || Last->ReorderIndices.empty()) && + "Reordering isn't implemented for non-power-of-2 nodes yet"); + } return Last; } @@ -4256,6 +4270,13 @@ BoUpSLP::LoadsState BoUpSLP::canVectorizeLoads( auto *VecTy = FixedVectorType::get(ScalarTy, Sz); // Check the order of pointer operands or that all pointers are the same. bool IsSorted = sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, Order); + // FIXME: Reordering isn't implemented for non-power-of-2 nodes yet. + if (!Order.empty() && !isPowerOf2_32(VL.size())) { + assert(VectorizeNonPowerOf2 && "non-power-of-2 number of loads only " + "supported with VectorizeNonPowerOf2"); + return LoadsState::Gather; + } + Align CommonAlignment = computeCommonAlignment(VL); if (!IsSorted && Sz > MinProfitableStridedLoads && TTI->isTypeLegal(VecTy) && TTI->isLegalStridedLoadStore(VecTy, CommonAlignment) && @@ -4575,6 +4596,10 @@ static bool areTwoInsertFromSameBuildVector( std::optional BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { + // FIXME: Vectorizing is not supported yet for non-power-of-2 ops. + if (TE.isNonPowOf2Vec()) + return std::nullopt; + // No need to reorder if need to shuffle reuses, still need to shuffle the // node. if (!TE.ReuseShuffleIndices.empty()) { @@ -5145,6 +5170,10 @@ bool BoUpSLP::canReorderOperands( TreeEntry *UserTE, SmallVectorImpl> &Edges, ArrayRef ReorderableGathers, SmallVectorImpl &GatherOps) { + // FIXME: Reordering isn't implemented for non-power-of-2 nodes yet. + if (UserTE->isNonPowOf2Vec()) + return false; + for (unsigned I = 0, E = UserTE->getNumOperands(); I < E; ++I) { if (any_of(Edges, [I](const std::pair &OpData) { return OpData.first == I && @@ -5318,6 +5347,9 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) { } auto Res = OrdersUses.insert(std::make_pair(OrdersType(), 0)); const auto AllowsReordering = [&](const TreeEntry *TE) { + // FIXME: Reordering isn't implemented for non-power-of-2 nodes yet. + if (TE->isNonPowOf2Vec()) + return false; if (!TE->ReorderIndices.empty() || !TE->ReuseShuffleIndices.empty() || (TE->State == TreeEntry::Vectorize && TE->isAltShuffle()) || (IgnoreReorder && TE->Idx == 0)) @@ -5944,6 +5976,9 @@ BoUpSLP::TreeEntry::EntryState BoUpSLP::getScalarsVectorizationState( case Instruction::ExtractValue: case Instruction::ExtractElement: { bool Reuse = canReuseExtract(VL, VL0, CurrentOrder); + // FIXME: Vectorizing is not supported yet for non-power-of-2 ops. + if (!isPowerOf2_32(VL.size())) + return TreeEntry::NeedToGather; if (Reuse || !CurrentOrder.empty()) return TreeEntry::Vectorize; LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n"); @@ -6258,6 +6293,13 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, if (NumUniqueScalarValues == VL.size()) { ReuseShuffleIndicies.clear(); } else { + // FIXME: Reshuffing scalars is not supported yet for non-power-of-2 ops. + if (UserTreeIdx.UserTE && UserTreeIdx.UserTE->isNonPowOf2Vec()) { + LLVM_DEBUG(dbgs() << "SLP: Reshuffling scalars not yet supported " + "for nodes with padding.\n"); + newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx); + return false; + } LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n"); if (NumUniqueScalarValues <= 1 || (UniquePositions.size() == 1 && all_of(UniqueValues, @@ -7868,7 +7910,8 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { for (unsigned I = 0, End = VL.size(); I < End; I += VF) { if (VectorizedLoads.contains(VL[I])) continue; - GatherCost += getBuildVectorCost(VL.slice(I, VF), Root); + GatherCost += + getBuildVectorCost(VL.slice(I, std::min(End - I, VF)), Root); } // Exclude potentially vectorized loads from list of gathered // scalars. @@ -10678,6 +10721,9 @@ BoUpSLP::isGatherShuffledEntry( // No need to check for the topmost gather node. if (TE == VectorizableTree.front().get()) return {}; + // FIXME: Gathering for non-power-of-2 nodes not implemented yet. + if (TE->isNonPowOf2Vec()) + return {}; Mask.assign(VL.size(), PoisonMaskElem); assert(TE->UserTreeIndices.size() == 1 && "Expected only single user of the gather node."); @@ -14995,8 +15041,13 @@ bool SLPVectorizerPass::vectorizeStoreChain(ArrayRef Chain, BoUpSLP &R, const unsigned Sz = R.getVectorElementSize(Chain[0]); unsigned VF = Chain.size(); - if (!isPowerOf2_32(Sz) || !isPowerOf2_32(VF) || VF < 2 || VF < MinVF) - return false; + if (!isPowerOf2_32(Sz) || !isPowerOf2_32(VF) || VF < 2 || VF < MinVF) { + // Check if vectorizing with a non-power-of-2 VF should be considered. At + // the moment, only consider cases where VF + 1 is a power-of-2, i.e. almost + // all vector lanes are used. + if (!VectorizeNonPowerOf2 || (VF < MinVF && VF + 1 != MinVF)) + return false; + } LLVM_DEBUG(dbgs() << "SLP: Analyzing " << VF << " stores at offset " << Idx << "\n"); @@ -15095,14 +15146,22 @@ bool SLPVectorizerPass::vectorizeStores(ArrayRef Stores, continue; } + unsigned NonPowerOf2VF = 0; + if (VectorizeNonPowerOf2) { + // First try vectorizing with a non-power-of-2 VF. At the moment, only + // consider cases where VF + 1 is a power-of-2, i.e. almost all vector + // lanes are used. + unsigned CandVF = Operands.size(); + if (isPowerOf2_32(CandVF + 1) && CandVF <= MaxVF) + NonPowerOf2VF = CandVF; + } + unsigned Sz = 1 + Log2_32(MaxVF) - Log2_32(MinVF); - SmallVector CandidateVFs(Sz); - // FIXME: Is division-by-2 the correct step? Should we assert that the - // register size is a power-of-2? - unsigned Size = MaxVF; - for_each(CandidateVFs, [&](unsigned &VF) { - VF = Size; - Size /= 2; + SmallVector CandidateVFs(Sz + (NonPowerOf2VF > 0 ? 1 : 0)); + unsigned Size = MinVF; + for_each(reverse(CandidateVFs), [&](unsigned &VF) { + VF = Size > MaxVF ? NonPowerOf2VF : Size; + Size *= 2; }); unsigned StartIdx = 0; for (unsigned Size : CandidateVFs) { diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll index b9e959d50befd..7b27489782fc4 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec15-base.ll @@ -1,35 +1,45 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=POW2-ONLY %s define void @v15_load_i8_mul_by_constant_store(ptr %src, ptr noalias %dst) { -; CHECK-LABEL: define void @v15_load_i8_mul_by_constant_store( -; CHECK-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 0 -; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[GEP_SRC_0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <8 x i8> [[TMP0]], -; CHECK-NEXT: store <8 x i8> [[TMP1]], ptr [[DST]], align 1 -; CHECK-NEXT: [[GEP_SRC_8:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 8 -; CHECK-NEXT: [[DST_8:%.*]] = getelementptr i8, ptr [[DST]], i8 8 -; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_SRC_8]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = mul nsw <4 x i8> [[TMP2]], -; CHECK-NEXT: store <4 x i8> [[TMP3]], ptr [[DST_8]], align 1 -; CHECK-NEXT: [[GEP_SRC_12:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 12 -; CHECK-NEXT: [[L_SRC_12:%.*]] = load i8, ptr [[GEP_SRC_12]], align 4 -; CHECK-NEXT: [[MUL_12:%.*]] = mul nsw i8 [[L_SRC_12]], 10 -; CHECK-NEXT: [[DST_12:%.*]] = getelementptr i8, ptr [[DST]], i8 12 -; CHECK-NEXT: store i8 [[MUL_12]], ptr [[DST_12]], align 1 -; CHECK-NEXT: [[GEP_SRC_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 13 -; CHECK-NEXT: [[L_SRC_13:%.*]] = load i8, ptr [[GEP_SRC_13]], align 4 -; CHECK-NEXT: [[MUL_13:%.*]] = mul nsw i8 [[L_SRC_13]], 10 -; CHECK-NEXT: [[DST_13:%.*]] = getelementptr i8, ptr [[DST]], i8 13 -; CHECK-NEXT: store i8 [[MUL_13]], ptr [[DST_13]], align 1 -; CHECK-NEXT: [[GEP_SRC_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 14 -; CHECK-NEXT: [[L_SRC_14:%.*]] = load i8, ptr [[GEP_SRC_14]], align 4 -; CHECK-NEXT: [[MUL_14:%.*]] = mul nsw i8 [[L_SRC_14]], 10 -; CHECK-NEXT: [[DST_14:%.*]] = getelementptr i8, ptr [[DST]], i8 14 -; CHECK-NEXT: store i8 [[MUL_14]], ptr [[DST_14]], align 1 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @v15_load_i8_mul_by_constant_store( +; NON-POW2-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = load <15 x i8>, ptr [[GEP_SRC_0]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = mul nsw <15 x i8> [[TMP0]], +; NON-POW2-NEXT: store <15 x i8> [[TMP1]], ptr [[DST]], align 1 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @v15_load_i8_mul_by_constant_store( +; POW2-ONLY-SAME: ptr [[SRC:%.*]], ptr noalias [[DST:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 0 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <8 x i8>, ptr [[GEP_SRC_0]], align 4 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = mul nsw <8 x i8> [[TMP0]], +; POW2-ONLY-NEXT: store <8 x i8> [[TMP1]], ptr [[DST]], align 1 +; POW2-ONLY-NEXT: [[GEP_SRC_8:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 8 +; POW2-ONLY-NEXT: [[DST_8:%.*]] = getelementptr i8, ptr [[DST]], i8 8 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[GEP_SRC_8]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = mul nsw <4 x i8> [[TMP2]], +; POW2-ONLY-NEXT: store <4 x i8> [[TMP3]], ptr [[DST_8]], align 1 +; POW2-ONLY-NEXT: [[GEP_SRC_12:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 12 +; POW2-ONLY-NEXT: [[L_SRC_12:%.*]] = load i8, ptr [[GEP_SRC_12]], align 4 +; POW2-ONLY-NEXT: [[MUL_12:%.*]] = mul nsw i8 [[L_SRC_12]], 10 +; POW2-ONLY-NEXT: [[DST_12:%.*]] = getelementptr i8, ptr [[DST]], i8 12 +; POW2-ONLY-NEXT: store i8 [[MUL_12]], ptr [[DST_12]], align 1 +; POW2-ONLY-NEXT: [[GEP_SRC_13:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 13 +; POW2-ONLY-NEXT: [[L_SRC_13:%.*]] = load i8, ptr [[GEP_SRC_13]], align 4 +; POW2-ONLY-NEXT: [[MUL_13:%.*]] = mul nsw i8 [[L_SRC_13]], 10 +; POW2-ONLY-NEXT: [[DST_13:%.*]] = getelementptr i8, ptr [[DST]], i8 13 +; POW2-ONLY-NEXT: store i8 [[MUL_13]], ptr [[DST_13]], align 1 +; POW2-ONLY-NEXT: [[GEP_SRC_14:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i8 14 +; POW2-ONLY-NEXT: [[L_SRC_14:%.*]] = load i8, ptr [[GEP_SRC_14]], align 4 +; POW2-ONLY-NEXT: [[MUL_14:%.*]] = mul nsw i8 [[L_SRC_14]], 10 +; POW2-ONLY-NEXT: [[DST_14:%.*]] = getelementptr i8, ptr [[DST]], i8 14 +; POW2-ONLY-NEXT: store i8 [[MUL_14]], ptr [[DST_14]], align 1 +; POW2-ONLY-NEXT: ret void ; entry: %gep.src.0 = getelementptr inbounds i8, ptr %src, i8 0 @@ -123,5 +133,3 @@ entry: ret void } - - diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll index 59ffbf7ef9b24..c18811a35c1ee 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-base.ll @@ -1,16 +1,69 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) { -; CHECK-LABEL: @v3_load_i32_mul_by_constant_store( +; NON-POW2-LABEL: @v3_load_i32_mul_by_constant_store( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_0]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = mul nsw <3 x i32> [[TMP0]], +; NON-POW2-NEXT: store <3 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @v3_load_i32_mul_by_constant_store( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4 +; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_0]], align 4 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], +; POW2-ONLY-NEXT: store <2 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 +; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2 +; POW2-ONLY-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 4 +; POW2-ONLY-NEXT: ret void +; +entry: + %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0 + %l.src.0 = load i32, ptr %gep.src.0, align 4 + %mul.0 = mul nsw i32 %l.src.0, 10 + + %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 1 + %l.src.1 = load i32, ptr %gep.src.1, align 4 + %mul.1 = mul nsw i32 %l.src.1, 10 + + %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 2 + %l.src.2 = load i32, ptr %gep.src.2, align 4 + %mul.2 = mul nsw i32 %l.src.2, 10 + + store i32 %mul.0, ptr %dst + + %dst.1 = getelementptr i32, ptr %dst, i32 1 + store i32 %mul.1, ptr %dst.1 + + %dst.2 = getelementptr i32, ptr %dst, i32 2 + store i32 %mul.2, ptr %dst.2 + + ret void +} + +; Should no be vectorized with a undef/poison element as padding, as division by undef/poison may cause UB. +define void @v3_load_i32_udiv_by_constant_store(ptr %src, ptr %dst) { +; CHECK-LABEL: @v3_load_i32_udiv_by_constant_store( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i32 0 +; CHECK-NEXT: [[L_SRC_0:%.*]] = load i32, ptr [[GEP_SRC_0]], align 4 +; CHECK-NEXT: [[MUL_0:%.*]] = udiv i32 10, [[L_SRC_0]] +; CHECK-NEXT: [[GEP_SRC_1:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 1 +; CHECK-NEXT: [[L_SRC_1:%.*]] = load i32, ptr [[GEP_SRC_1]], align 4 +; CHECK-NEXT: [[MUL_1:%.*]] = udiv i32 10, [[L_SRC_1]] ; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i32 2 ; CHECK-NEXT: [[L_SRC_2:%.*]] = load i32, ptr [[GEP_SRC_2]], align 4 -; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_2]], 10 -; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], -; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[MUL_2:%.*]] = udiv i32 10, [[L_SRC_2]] +; CHECK-NEXT: store i32 [[MUL_0]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[DST_1:%.*]] = getelementptr i32, ptr [[DST]], i32 1 +; CHECK-NEXT: store i32 [[MUL_1]], ptr [[DST_1]], align 4 ; CHECK-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2 ; CHECK-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 4 ; CHECK-NEXT: ret void @@ -18,15 +71,15 @@ define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) { entry: %gep.src.0 = getelementptr inbounds i32, ptr %src, i32 0 %l.src.0 = load i32, ptr %gep.src.0, align 4 - %mul.0 = mul nsw i32 %l.src.0, 10 + %mul.0 = udiv i32 10, %l.src.0 %gep.src.1 = getelementptr inbounds i32, ptr %src, i32 1 %l.src.1 = load i32, ptr %gep.src.1, align 4 - %mul.1 = mul nsw i32 %l.src.1, 10 + %mul.1 = udiv i32 10, %l.src.1 %gep.src.2 = getelementptr inbounds i32, ptr %src, i32 2 %l.src.2 = load i32, ptr %gep.src.2, align 4 - %mul.2 = mul nsw i32 %l.src.2, 10 + %mul.2 = udiv i32 10, %l.src.2 store i32 %mul.0, ptr %dst @@ -39,23 +92,35 @@ entry: ret void } + + define void @v3_load_i32_mul_store(ptr %src.1, ptr %src.2, ptr %dst) { -; CHECK-LABEL: @v3_load_i32_mul_store( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0 -; CHECK-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0 -; CHECK-NEXT: [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2 -; CHECK-NEXT: [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4 -; CHECK-NEXT: [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2 -; CHECK-NEXT: [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4 -; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]] -; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]] -; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 -; CHECK-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2 -; CHECK-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @v3_load_i32_mul_store( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0 +; NON-POW2-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_1_0]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_2_0]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP0]], [[TMP1]] +; NON-POW2-NEXT: store <3 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @v3_load_i32_mul_store( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4 +; POW2-ONLY-NEXT: [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4 +; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]] +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]] +; POW2-ONLY-NEXT: store <2 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 +; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2 +; POW2-ONLY-NEXT: store i32 [[MUL_2]], ptr [[DST_2]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %gep.src.1.0 = getelementptr inbounds i32, ptr %src.1, i32 0 @@ -88,24 +153,35 @@ entry: } define void @v3_load_i32_mul_add_const_store(ptr %src.1, ptr %src.2, ptr %dst) { -; CHECK-LABEL: @v3_load_i32_mul_add_const_store( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0 -; CHECK-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0 -; CHECK-NEXT: [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2 -; CHECK-NEXT: [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4 -; CHECK-NEXT: [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2 -; CHECK-NEXT: [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4 -; CHECK-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]] -; CHECK-NEXT: [[ADD_2:%.*]] = add i32 [[MUL_2]], 9 -; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]] -; CHECK-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP2]], -; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[DST:%.*]], align 4 -; CHECK-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2 -; CHECK-NEXT: store i32 [[ADD_2]], ptr [[DST_2]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @v3_load_i32_mul_add_const_store( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0 +; NON-POW2-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr [[GEP_SRC_1_0]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[GEP_SRC_2_0]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = mul nsw <3 x i32> [[TMP0]], [[TMP1]] +; NON-POW2-NEXT: [[TMP3:%.*]] = add <3 x i32> [[TMP2]], +; NON-POW2-NEXT: store <3 x i32> [[TMP3]], ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @v3_load_i32_mul_add_const_store( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[GEP_SRC_1_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_1:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_2_0:%.*]] = getelementptr inbounds i32, ptr [[SRC_2:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_1_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_1]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_1_2:%.*]] = load i32, ptr [[GEP_SRC_1_2]], align 4 +; POW2-ONLY-NEXT: [[GEP_SRC_2_2:%.*]] = getelementptr inbounds i32, ptr [[SRC_2]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_2_2:%.*]] = load i32, ptr [[GEP_SRC_2_2]], align 4 +; POW2-ONLY-NEXT: [[MUL_2:%.*]] = mul nsw i32 [[L_SRC_1_2]], [[L_SRC_2_2]] +; POW2-ONLY-NEXT: [[ADD_2:%.*]] = add i32 [[MUL_2]], 9 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[GEP_SRC_1_0]], align 4 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP_SRC_2_0]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP0]], [[TMP1]] +; POW2-ONLY-NEXT: [[TMP3:%.*]] = add <2 x i32> [[TMP2]], +; POW2-ONLY-NEXT: store <2 x i32> [[TMP3]], ptr [[DST:%.*]], align 4 +; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST]], i32 2 +; POW2-ONLY-NEXT: store i32 [[ADD_2]], ptr [[DST_2]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %gep.src.1.0 = getelementptr inbounds i32, ptr %src.1, i32 0 @@ -141,18 +217,26 @@ entry: } define void @v3_load_f32_fadd_fadd_by_constant_store(ptr %src, ptr %dst) { -; CHECK-LABEL: @v3_load_f32_fadd_fadd_by_constant_store( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0 -; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2 -; CHECK-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4 -; CHECK-NEXT: [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01 -; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], -; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4 -; CHECK-NEXT: [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2 -; CHECK-NEXT: store float [[FADD_2]], ptr [[DST_2]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @v3_load_f32_fadd_fadd_by_constant_store( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = fadd <3 x float> [[TMP0]], +; NON-POW2-NEXT: store <3 x float> [[TMP1]], ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @v3_load_f32_fadd_fadd_by_constant_store( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4 +; POW2-ONLY-NEXT: [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], +; POW2-ONLY-NEXT: store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4 +; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2 +; POW2-ONLY-NEXT: store float [[FADD_2]], ptr [[DST_2]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %gep.src.0 = getelementptr inbounds float, ptr %src, i32 0 @@ -179,18 +263,28 @@ entry: } define void @phi_store3(ptr %dst) { -; CHECK-LABEL: @phi_store3( -; CHECK-NEXT: entry: -; CHECK-NEXT: br label [[EXIT:%.*]] -; CHECK: invoke.cont8.loopexit: -; CHECK-NEXT: br label [[EXIT]] -; CHECK: exit: -; CHECK-NEXT: [[P_2:%.*]] = phi i32 [ 3, [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT8_LOOPEXIT:%.*]] ] -; CHECK-NEXT: [[TMP0:%.*]] = phi <2 x i32> [ , [[ENTRY]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT]] ] -; CHECK-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 2 -; CHECK-NEXT: store <2 x i32> [[TMP0]], ptr [[DST]], align 4 -; CHECK-NEXT: store i32 [[P_2]], ptr [[DST_2]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @phi_store3( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: br label [[EXIT:%.*]] +; NON-POW2: invoke.cont8.loopexit: +; NON-POW2-NEXT: br label [[EXIT]] +; NON-POW2: exit: +; NON-POW2-NEXT: [[TMP0:%.*]] = phi <3 x i32> [ , [[ENTRY:%.*]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT:%.*]] ] +; NON-POW2-NEXT: store <3 x i32> [[TMP0]], ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @phi_store3( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: br label [[EXIT:%.*]] +; POW2-ONLY: invoke.cont8.loopexit: +; POW2-ONLY-NEXT: br label [[EXIT]] +; POW2-ONLY: exit: +; POW2-ONLY-NEXT: [[P_2:%.*]] = phi i32 [ 3, [[ENTRY:%.*]] ], [ 0, [[INVOKE_CONT8_LOOPEXIT:%.*]] ] +; POW2-ONLY-NEXT: [[TMP0:%.*]] = phi <2 x i32> [ , [[ENTRY]] ], [ poison, [[INVOKE_CONT8_LOOPEXIT]] ] +; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr i32, ptr [[DST:%.*]], i32 2 +; POW2-ONLY-NEXT: store <2 x i32> [[TMP0]], ptr [[DST]], align 4 +; POW2-ONLY-NEXT: store i32 [[P_2]], ptr [[DST_2]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: br label %exit @@ -213,13 +307,18 @@ exit: } define void @store_try_reorder(ptr %dst) { -; CHECK-LABEL: @store_try_reorder( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[ADD:%.*]] = add i32 0, 0 -; CHECK-NEXT: store i32 [[ADD]], ptr [[DST:%.*]], align 4 -; CHECK-NEXT: [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1 -; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @store_try_reorder( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: store <3 x i32> zeroinitializer, ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @store_try_reorder( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[ADD:%.*]] = add i32 0, 0 +; POW2-ONLY-NEXT: store i32 [[ADD]], ptr [[DST:%.*]], align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX_I1887:%.*]] = getelementptr i32, ptr [[DST]], i64 1 +; POW2-ONLY-NEXT: store <2 x i32> zeroinitializer, ptr [[ARRAYIDX_I1887]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %add = add i32 0, 0 diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll index 2cb84eeb7fc8f..67746f2cbf5d2 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-calls.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK %s define void @vec3_vectorize_call(ptr %Colour, float %0) { ; CHECK-LABEL: @vec3_vectorize_call( diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll index e4925961f284f..47d918eabdfe2 100644 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll +++ b/llvm/test/Transforms/SLPVectorizer/AArch64/vec3-reorder-reshuffle.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -passes=slp-vectorizer -mtriple=arm64-apple-ios -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=arm64-apple-ios -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s %struct.zot = type { i32, i32, i32 } @@ -139,21 +140,34 @@ if.end668: ; preds = %if.then665, %entry } define void @gather_2(ptr %mat1, float %0, float %1) { -; CHECK-LABEL: define void @gather_2( -; CHECK-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x float> , float [[TMP1]], i32 1 -; CHECK-NEXT: [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP6]], <2 x float> zeroinitializer) -; CHECK-NEXT: [[TMP4:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00) -; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP4]], 0.000000e+00 -; CHECK-NEXT: [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1 -; CHECK-NEXT: [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2 -; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer -; CHECK-NEXT: store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4 -; CHECK-NEXT: store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @gather_2( +; NON-POW2-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0 +; NON-POW2-NEXT: [[TMP3:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <3 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP4:%.*]] = insertelement <3 x float> , float [[TMP1]], i32 1 +; NON-POW2-NEXT: [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP6:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP3]], <3 x float> [[TMP5]], <3 x float> zeroinitializer) +; NON-POW2-NEXT: [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1 +; NON-POW2-NEXT: [[TMP7:%.*]] = fmul <3 x float> [[TMP6]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP7]], ptr [[ARRAYIDX163]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @gather_2( +; POW2-ONLY-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP4:%.*]] = insertelement <2 x float> , float [[TMP1]], i32 1 +; POW2-ONLY-NEXT: [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: [[TMP6:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00) +; POW2-ONLY-NEXT: [[TMP7:%.*]] = fmul float [[TMP6]], 0.000000e+00 +; POW2-ONLY-NEXT: [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1 +; POW2-ONLY-NEXT: [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2 +; POW2-ONLY-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer +; POW2-ONLY-NEXT: store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4 +; POW2-ONLY-NEXT: store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %2 = call float @llvm.fmuladd.f32(float %0, float 0.000000e+00, float 0.000000e+00) @@ -172,31 +186,47 @@ entry: } define i32 @reorder_indices_1(float %0) { -; CHECK-LABEL: define i32 @reorder_indices_1( -; CHECK-SAME: float [[TMP0:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4 -; CHECK-NEXT: [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]] -; CHECK-NEXT: [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]] -; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]]) -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = fneg <2 x float> [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]]) -; CHECK-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer) -; CHECK-NEXT: [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00) -; CHECK-NEXT: [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer -; CHECK-NEXT: [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00 -; CHECK-NEXT: store <2 x float> [[TMP15]], ptr [[NOR1]], align 4 -; CHECK-NEXT: store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4 -; CHECK-NEXT: ret i32 0 +; NON-POW2-LABEL: define i32 @reorder_indices_1( +; NON-POW2-SAME: float [[TMP0:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[NOR1]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP3:%.*]] = fneg <3 x float> [[TMP2]] +; NON-POW2-NEXT: [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0 +; NON-POW2-NEXT: [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]] +; NON-POW2-NEXT: [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP6]]) +; NON-POW2-NEXT: [[TMP8:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> [[TMP7]], <3 x float> zeroinitializer) +; NON-POW2-NEXT: [[TMP9:%.*]] = fmul <3 x float> [[TMP8]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP9]], ptr [[NOR1]], align 4 +; NON-POW2-NEXT: ret i32 0 +; +; POW2-ONLY-LABEL: define i32 @reorder_indices_1( +; POW2-ONLY-SAME: float [[TMP0:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0 +; POW2-ONLY-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]] +; POW2-ONLY-NEXT: [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]] +; POW2-ONLY-NEXT: [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]]) +; POW2-ONLY-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> +; POW2-ONLY-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1 +; POW2-ONLY-NEXT: [[TMP8:%.*]] = fneg <2 x float> [[TMP7]] +; POW2-ONLY-NEXT: [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]] +; POW2-ONLY-NEXT: [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]]) +; POW2-ONLY-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00) +; POW2-ONLY-NEXT: [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer +; POW2-ONLY-NEXT: [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00 +; POW2-ONLY-NEXT: store <2 x float> [[TMP15]], ptr [[NOR1]], align 4 +; POW2-ONLY-NEXT: store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4 +; POW2-ONLY-NEXT: ret i32 0 ; entry: %nor1 = alloca [0 x [3 x float]], i32 0, align 4 @@ -227,19 +257,28 @@ entry: } define void @reorder_indices_2(ptr %spoint) { -; CHECK-LABEL: define void @reorder_indices_2( -; CHECK-SAME: ptr [[SPOINT:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0 -; CHECK-NEXT: [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00) -; CHECK-NEXT: [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00 -; CHECK-NEXT: [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0 -; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer) -; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer -; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[DSCO]], align 4 -; CHECK-NEXT: [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2 -; CHECK-NEXT: store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @reorder_indices_2( +; NON-POW2-SAME: ptr [[SPOINT:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> zeroinitializer, <3 x float> zeroinitializer, <3 x float> zeroinitializer) +; NON-POW2-NEXT: [[TMP1:%.*]] = fmul <3 x float> [[TMP0]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP1]], ptr [[DSCO]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @reorder_indices_2( +; POW2-ONLY-SAME: ptr [[SPOINT:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00) +; POW2-ONLY-NEXT: [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00 +; POW2-ONLY-NEXT: [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer +; POW2-ONLY-NEXT: store <2 x float> [[TMP3]], ptr [[DSCO]], align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2 +; POW2-ONLY-NEXT: store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %0 = extractelement <3 x float> zeroinitializer, i64 1 @@ -291,19 +330,30 @@ entry: } define void @reuse_shuffle_indidces_1(ptr %col, float %0, float %1) { -; CHECK-LABEL: define void @reuse_shuffle_indidces_1( -; CHECK-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer -; CHECK-NEXT: store <2 x float> [[TMP5]], ptr [[COL]], align 4 -; CHECK-NEXT: [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2 -; CHECK-NEXT: [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00 -; CHECK-NEXT: [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00 -; CHECK-NEXT: store float [[TMP6]], ptr [[ARRAYIDX33]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @reuse_shuffle_indidces_1( +; NON-POW2-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP1]], i32 0 +; NON-POW2-NEXT: [[TMP3:%.*]] = insertelement <3 x float> [[TMP2]], float [[TMP0]], i32 1 +; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP5:%.*]] = fmul <3 x float> [[TMP4]], zeroinitializer +; NON-POW2-NEXT: [[TMP6:%.*]] = fadd <3 x float> [[TMP5]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP6]], ptr [[COL]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @reuse_shuffle_indidces_1( +; POW2-ONLY-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1 +; POW2-ONLY-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer +; POW2-ONLY-NEXT: [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer +; POW2-ONLY-NEXT: store <2 x float> [[TMP5]], ptr [[COL]], align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2 +; POW2-ONLY-NEXT: [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00 +; POW2-ONLY-NEXT: [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00 +; POW2-ONLY-NEXT: store float [[TMP6]], ptr [[ARRAYIDX33]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %mul24 = fmul float %1, 0.000000e+00 @@ -487,18 +537,24 @@ entry: } define void @vec3_extract(<3 x i16> %pixel.sroa.0.4.vec.insert606, ptr %call3.i536) { -; CHECK-LABEL: define void @vec3_extract( -; CHECK-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2 -; CHECK-NEXT: [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2 -; CHECK-NEXT: store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2 -; CHECK-NEXT: [[PIXEL_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 1 -; CHECK-NEXT: [[GREEN670:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 1 -; CHECK-NEXT: store i16 [[PIXEL_SROA_0_2_VEC_EXTRACT]], ptr [[GREEN670]], align 2 -; CHECK-NEXT: [[PIXEL_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 0 -; CHECK-NEXT: store i16 [[PIXEL_SROA_0_0_VEC_EXTRACT]], ptr [[CALL3_I536]], align 2 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @vec3_extract( +; NON-POW2-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: store <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], ptr [[CALL3_I536]], align 2 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @vec3_extract( +; POW2-ONLY-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2 +; POW2-ONLY-NEXT: [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2 +; POW2-ONLY-NEXT: store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2 +; POW2-ONLY-NEXT: [[PIXEL_SROA_0_2_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 1 +; POW2-ONLY-NEXT: [[GREEN670:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 1 +; POW2-ONLY-NEXT: store i16 [[PIXEL_SROA_0_2_VEC_EXTRACT]], ptr [[GREEN670]], align 2 +; POW2-ONLY-NEXT: [[PIXEL_SROA_0_0_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 0 +; POW2-ONLY-NEXT: store i16 [[PIXEL_SROA_0_0_VEC_EXTRACT]], ptr [[CALL3_I536]], align 2 +; POW2-ONLY-NEXT: ret void ; entry: %pixel.sroa.0.4.vec.extract = extractelement <3 x i16> %pixel.sroa.0.4.vec.insert606, i64 2 @@ -512,4 +568,56 @@ entry: ret void } +define void @can_reorder_vec3_op_with_padding(ptr %A, <3 x float> %in) { +; NON-POW2-LABEL: define void @can_reorder_vec3_op_with_padding( +; NON-POW2-SAME: ptr [[A:%.*]], <3 x float> [[IN:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[TMP0:%.*]] = shufflevector <3 x float> [[IN]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP1:%.*]] = fsub <3 x float> [[TMP0]], [[TMP0]] +; NON-POW2-NEXT: [[TMP2:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> , <3 x float> ) +; NON-POW2-NEXT: [[TMP3:%.*]] = fmul <3 x float> [[TMP2]], +; NON-POW2-NEXT: store <3 x float> [[TMP3]], ptr [[A]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @can_reorder_vec3_op_with_padding( +; POW2-ONLY-SAME: ptr [[A:%.*]], <3 x float> [[IN:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[ARRAYIDX42_I:%.*]] = getelementptr float, ptr [[A]], i64 2 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = extractelement <3 x float> [[IN]], i64 0 +; POW2-ONLY-NEXT: [[SUB_I362:%.*]] = fsub float [[TMP0]], [[TMP0]] +; POW2-ONLY-NEXT: [[TMP1:%.*]] = call float @llvm.fmuladd.f32(float [[SUB_I362]], float 2.000000e+00, float 3.000000e+00) +; POW2-ONLY-NEXT: [[MUL6_I_I_I_I:%.*]] = fmul float [[TMP1]], 3.000000e+00 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[IN]], <3 x float> poison, <2 x i32> +; POW2-ONLY-NEXT: [[TMP3:%.*]] = fsub <2 x float> [[TMP2]], [[TMP2]] +; POW2-ONLY-NEXT: [[TMP4:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> , <2 x float> ) +; POW2-ONLY-NEXT: [[TMP5:%.*]] = fmul <2 x float> [[TMP4]], +; POW2-ONLY-NEXT: store <2 x float> [[TMP5]], ptr [[A]], align 4 +; POW2-ONLY-NEXT: store float [[MUL6_I_I_I_I]], ptr [[ARRAYIDX42_I]], align 4 +; POW2-ONLY-NEXT: ret void +; +entry: + %arrayidx42.i = getelementptr float, ptr %A, i64 2 + %arrayidx35.i = getelementptr float, ptr %A, i64 1 + %0 = extractelement <3 x float> %in, i64 0 + %1 = extractelement <3 x float> %in, i64 0 + %sub.i362 = fsub float %0, %1 + %2 = extractelement <3 x float> %in, i64 1 + %3 = extractelement <3 x float> %in, i64 1 + %sub5.i = fsub float %2, %3 + %4 = extractelement <3 x float> %in, i64 2 + %5 = extractelement <3 x float> %in, i64 2 + %sub9.i = fsub float %4, %5 + %6 = call float @llvm.fmuladd.f32(float %sub5.i, float 2.000000e+00, float 3.000000e+00) + %7 = call float @llvm.fmuladd.f32(float %sub9.i, float 2.000000e+00, float 3.000000e+00) + %8 = call float @llvm.fmuladd.f32(float %sub.i362, float 2.000000e+00, float 3.000000e+00) + %mul.i.i.i.i373 = fmul float %6, 3.000000e+00 + %mul3.i.i.i.i = fmul float %7, 3.000000e+00 + %mul6.i.i.i.i = fmul float %8, 3.000000e+00 + store float %mul.i.i.i.i373, ptr %A, align 4 + store float %mul3.i.i.i.i, ptr %arrayidx35.i, align 4 + store float %mul6.i.i.i.i, ptr %arrayidx42.i, align 4 + ret void +} + declare float @llvm.fmuladd.f32(float, float, float) +declare double @llvm.fmuladd.f64(double, double, double) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll index 4795ac6559203..5f2c42d5c2dec 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/odd_store.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt < %s -passes=slp-vectorizer,dce -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck %s +; RUN: opt < %s -passes=slp-vectorizer,dce -slp-vectorize-non-power-of-2 -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt < %s -passes=slp-vectorizer,dce -slp-vectorize-non-power-of-2=false -S -mtriple=x86_64-apple-macosx10.8.0 -mcpu=corei7-avx | FileCheck --check-prefixes=CHECK,POW2-ONLY %s ;int foo(char * restrict A, ptr restrict B, float T) { ; A[0] = (T * B[10] + 4.0); @@ -8,31 +9,43 @@ ;} define i32 @foo(ptr noalias nocapture %A, ptr noalias nocapture %B, float %T) { -; CHECK-LABEL: @foo( -; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10 -; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[TMP1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]] -; CHECK-NEXT: [[TMP4:%.*]] = fpext float [[TMP3]] to double -; CHECK-NEXT: [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00 -; CHECK-NEXT: [[TMP6:%.*]] = fptosi double [[TMP5]] to i8 -; CHECK-NEXT: store i8 [[TMP6]], ptr [[A:%.*]], align 1 -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11 -; CHECK-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4 -; CHECK-NEXT: [[TMP9:%.*]] = fmul float [[TMP8]], [[T]] -; CHECK-NEXT: [[TMP10:%.*]] = fpext float [[TMP9]] to double -; CHECK-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 5.000000e+00 -; CHECK-NEXT: [[TMP12:%.*]] = fptosi double [[TMP11]] to i8 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1 -; CHECK-NEXT: store i8 [[TMP12]], ptr [[TMP13]], align 1 -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 12 -; CHECK-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4 -; CHECK-NEXT: [[TMP16:%.*]] = fmul float [[TMP15]], [[T]] -; CHECK-NEXT: [[TMP17:%.*]] = fpext float [[TMP16]] to double -; CHECK-NEXT: [[TMP18:%.*]] = fadd double [[TMP17]], 6.000000e+00 -; CHECK-NEXT: [[TMP19:%.*]] = fptosi double [[TMP18]] to i8 -; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2 -; CHECK-NEXT: store i8 [[TMP19]], ptr [[TMP20]], align 1 -; CHECK-NEXT: ret i32 undef +; NON-POW2-LABEL: @foo( +; NON-POW2-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10 +; NON-POW2-NEXT: [[TMP2:%.*]] = load <3 x float>, ptr [[TMP1]], align 4 +; NON-POW2-NEXT: [[TMP3:%.*]] = insertelement <3 x float> poison, float [[T:%.*]], i32 0 +; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP5:%.*]] = fmul <3 x float> [[TMP2]], [[TMP4]] +; NON-POW2-NEXT: [[TMP6:%.*]] = fpext <3 x float> [[TMP5]] to <3 x double> +; NON-POW2-NEXT: [[TMP7:%.*]] = fadd <3 x double> [[TMP6]], +; NON-POW2-NEXT: [[TMP8:%.*]] = fptosi <3 x double> [[TMP7]] to <3 x i8> +; NON-POW2-NEXT: store <3 x i8> [[TMP8]], ptr [[A:%.*]], align 1 +; NON-POW2-NEXT: ret i32 undef +; +; POW2-ONLY-LABEL: @foo( +; POW2-ONLY-NEXT: [[TMP1:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], i64 10 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = load float, ptr [[TMP1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = fmul float [[TMP2]], [[T:%.*]] +; POW2-ONLY-NEXT: [[TMP4:%.*]] = fpext float [[TMP3]] to double +; POW2-ONLY-NEXT: [[TMP5:%.*]] = fadd double [[TMP4]], 4.000000e+00 +; POW2-ONLY-NEXT: [[TMP6:%.*]] = fptosi double [[TMP5]] to i8 +; POW2-ONLY-NEXT: store i8 [[TMP6]], ptr [[A:%.*]], align 1 +; POW2-ONLY-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[B]], i64 11 +; POW2-ONLY-NEXT: [[TMP8:%.*]] = load float, ptr [[TMP7]], align 4 +; POW2-ONLY-NEXT: [[TMP9:%.*]] = fmul float [[TMP8]], [[T]] +; POW2-ONLY-NEXT: [[TMP10:%.*]] = fpext float [[TMP9]] to double +; POW2-ONLY-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 5.000000e+00 +; POW2-ONLY-NEXT: [[TMP12:%.*]] = fptosi double [[TMP11]] to i8 +; POW2-ONLY-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 1 +; POW2-ONLY-NEXT: store i8 [[TMP12]], ptr [[TMP13]], align 1 +; POW2-ONLY-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[B]], i64 12 +; POW2-ONLY-NEXT: [[TMP15:%.*]] = load float, ptr [[TMP14]], align 4 +; POW2-ONLY-NEXT: [[TMP16:%.*]] = fmul float [[TMP15]], [[T]] +; POW2-ONLY-NEXT: [[TMP17:%.*]] = fpext float [[TMP16]] to double +; POW2-ONLY-NEXT: [[TMP18:%.*]] = fadd double [[TMP17]], 6.000000e+00 +; POW2-ONLY-NEXT: [[TMP19:%.*]] = fptosi double [[TMP18]] to i8 +; POW2-ONLY-NEXT: [[TMP20:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 2 +; POW2-ONLY-NEXT: store i8 [[TMP19]], ptr [[TMP20]], align 1 +; POW2-ONLY-NEXT: ret i32 undef ; %1 = getelementptr inbounds float, ptr %B, i64 10 %2 = load float, ptr %1, align 4 @@ -91,13 +104,18 @@ define void @test_v4f32_v2f32_splat_store(<4 x float> %f, ptr %p){ } define void @test_v4f32_v3f32_store(<4 x float> %f, ptr %p){ -; CHECK-LABEL: @test_v4f32_v3f32_store( -; CHECK-NEXT: [[X2:%.*]] = extractelement <4 x float> [[F:%.*]], i64 2 -; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[F]], <4 x float> poison, <2 x i32> -; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[P]], align 4 -; CHECK-NEXT: store float [[X2]], ptr [[P2]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @test_v4f32_v3f32_store( +; NON-POW2-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[F:%.*]], <4 x float> poison, <3 x i32> +; NON-POW2-NEXT: store <3 x float> [[TMP1]], ptr [[P:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @test_v4f32_v3f32_store( +; POW2-ONLY-NEXT: [[X2:%.*]] = extractelement <4 x float> [[F:%.*]], i64 2 +; POW2-ONLY-NEXT: [[P2:%.*]] = getelementptr inbounds float, ptr [[P:%.*]], i64 2 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[F]], <4 x float> poison, <2 x i32> +; POW2-ONLY-NEXT: store <2 x float> [[TMP1]], ptr [[P]], align 4 +; POW2-ONLY-NEXT: store float [[X2]], ptr [[P2]], align 4 +; POW2-ONLY-NEXT: ret void ; %x0 = extractelement <4 x float> %f, i64 0 %x1 = extractelement <4 x float> %f, i64 1 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll index 6560fc6a14526..96d4b84e03691 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-base.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s define void @v3_load_i32_mul_by_constant_store(ptr %src, ptr %dst) { ; CHECK-LABEL: @v3_load_i32_mul_by_constant_store( @@ -161,18 +162,26 @@ entry: } define void @v3_load_f32_fadd_fadd_by_constant_store(ptr %src, ptr %dst) { -; CHECK-LABEL: @v3_load_f32_fadd_fadd_by_constant_store( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0 -; CHECK-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2 -; CHECK-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4 -; CHECK-NEXT: [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01 -; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4 -; CHECK-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], -; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4 -; CHECK-NEXT: [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2 -; CHECK-NEXT: store float [[FADD_2]], ptr [[DST_2]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @v3_load_f32_fadd_fadd_by_constant_store( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = load <3 x float>, ptr [[GEP_SRC_0]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = fadd <3 x float> [[TMP0]], +; NON-POW2-NEXT: store <3 x float> [[TMP1]], ptr [[DST:%.*]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @v3_load_f32_fadd_fadd_by_constant_store( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[GEP_SRC_0:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i32 0 +; POW2-ONLY-NEXT: [[GEP_SRC_2:%.*]] = getelementptr inbounds float, ptr [[SRC]], i32 2 +; POW2-ONLY-NEXT: [[L_SRC_2:%.*]] = load float, ptr [[GEP_SRC_2]], align 4 +; POW2-ONLY-NEXT: [[FADD_2:%.*]] = fadd float [[L_SRC_2]], 1.000000e+01 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[GEP_SRC_0]], align 4 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = fadd <2 x float> [[TMP0]], +; POW2-ONLY-NEXT: store <2 x float> [[TMP1]], ptr [[DST:%.*]], align 4 +; POW2-ONLY-NEXT: [[DST_2:%.*]] = getelementptr float, ptr [[DST]], i32 2 +; POW2-ONLY-NEXT: store float [[FADD_2]], ptr [[DST_2]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %gep.src.0 = getelementptr inbounds float, ptr %src, i32 0 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll index 71b9315839ecf..243087c6d8d95 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-calls.ll @@ -1,16 +1,29 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s define void @vec3_vectorize_call(ptr %Colour, float %0) { -; CHECK-LABEL: @vec3_vectorize_call( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[COLOUR:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP1]], <2 x float> zeroinitializer, <2 x float> zeroinitializer) -; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[COLOUR]], align 4 -; CHECK-NEXT: [[ARRAYIDX99_I1:%.*]] = getelementptr float, ptr [[COLOUR]], i64 2 -; CHECK-NEXT: [[TMP3:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0:%.*]], float 0.000000e+00, float 0.000000e+00) -; CHECK-NEXT: store float [[TMP3]], ptr [[ARRAYIDX99_I1]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @vec3_vectorize_call( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[TMP1:%.*]] = load float, ptr [[COLOUR:%.*]], align 4 +; NON-POW2-NEXT: [[ARRAYIDX91_I:%.*]] = getelementptr float, ptr [[COLOUR]], i64 1 +; NON-POW2-NEXT: [[TMP2:%.*]] = load float, ptr [[ARRAYIDX91_I]], align 4 +; NON-POW2-NEXT: [[TMP3:%.*]] = insertelement <3 x float> poison, float [[TMP0:%.*]], i32 2 +; NON-POW2-NEXT: [[TMP4:%.*]] = insertelement <3 x float> [[TMP3]], float [[TMP1]], i32 0 +; NON-POW2-NEXT: [[TMP5:%.*]] = insertelement <3 x float> [[TMP4]], float [[TMP2]], i32 1 +; NON-POW2-NEXT: [[TMP6:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> zeroinitializer, <3 x float> zeroinitializer) +; NON-POW2-NEXT: store <3 x float> [[TMP6]], ptr [[COLOUR]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @vec3_vectorize_call( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[COLOUR:%.*]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP1]], <2 x float> zeroinitializer, <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: store <2 x float> [[TMP2]], ptr [[COLOUR]], align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX99_I1:%.*]] = getelementptr float, ptr [[COLOUR]], i64 2 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0:%.*]], float 0.000000e+00, float 0.000000e+00) +; POW2-ONLY-NEXT: store float [[TMP3]], ptr [[ARRAYIDX99_I1]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %1 = load float, ptr %Colour, align 4 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll index 1411f9416f69d..e8adda0bdc703 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-gather-some-loads.ll @@ -1,35 +1,55 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=POW2-ONLY %s target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128" define void @test_insert_loads(ptr %A, ptr noalias %B, float %0) #0 { -; CHECK-LABEL: define void @test_insert_loads( -; CHECK-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], float [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[MULADD_0:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 1.000000e+00, float 1.000000e+00) -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP3:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> , <2 x float> ) -; CHECK-NEXT: [[A_28:%.*]] = getelementptr i8, ptr [[A]], i64 28 -; CHECK-NEXT: [[L_A_28:%.*]] = load float, ptr [[A_28]], align 4 -; CHECK-NEXT: [[A_12:%.*]] = getelementptr i8, ptr [[A]], i64 12 -; CHECK-NEXT: [[L_A_12:%.*]] = load float, ptr [[A_12]], align 4 -; CHECK-NEXT: [[GEP_4:%.*]] = getelementptr i8, ptr [[B]], i64 4 -; CHECK-NEXT: [[L_B_0:%.*]] = load float, ptr [[B]], align 4 -; CHECK-NEXT: [[GEP_28:%.*]] = getelementptr i8, ptr [[B]], i64 28 -; CHECK-NEXT: [[GEP_20:%.*]] = getelementptr i8, ptr [[B]], i64 20 -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> , float [[L_A_12]], i32 0 -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[L_A_28]], i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x float> , float [[L_B_0]], i32 0 -; CHECK-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP5]], <4 x float> [[TMP8]], <4 x float> [[TMP9]]) -; CHECK-NEXT: store <4 x float> [[TMP10]], ptr [[GEP_4]], align 4 -; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[GEP_20]], align 4 -; CHECK-NEXT: store float [[MULADD_0]], ptr [[GEP_28]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @test_insert_loads( +; NON-POW2-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], float [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[A_28:%.*]] = getelementptr i8, ptr [[A]], i64 28 +; NON-POW2-NEXT: [[L_A_28:%.*]] = load float, ptr [[A_28]], align 4 +; NON-POW2-NEXT: [[A_12:%.*]] = getelementptr i8, ptr [[A]], i64 12 +; NON-POW2-NEXT: [[L_A_12:%.*]] = load float, ptr [[A_12]], align 4 +; NON-POW2-NEXT: [[GEP_4:%.*]] = getelementptr i8, ptr [[B]], i64 4 +; NON-POW2-NEXT: [[L_B_0:%.*]] = load float, ptr [[B]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = insertelement <7 x float> poison, float [[TMP0]], i32 0 +; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <7 x float> [[TMP1]], <7 x float> poison, <7 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP3:%.*]] = insertelement <7 x float> , float [[L_A_12]], i32 0 +; NON-POW2-NEXT: [[TMP4:%.*]] = insertelement <7 x float> [[TMP3]], float [[L_A_28]], i32 1 +; NON-POW2-NEXT: [[TMP5:%.*]] = shufflevector <7 x float> [[TMP4]], <7 x float> poison, <7 x i32> +; NON-POW2-NEXT: [[TMP6:%.*]] = insertelement <7 x float> , float [[L_B_0]], i32 0 +; NON-POW2-NEXT: [[TMP7:%.*]] = call <7 x float> @llvm.fmuladd.v7f32(<7 x float> [[TMP2]], <7 x float> [[TMP5]], <7 x float> [[TMP6]]) +; NON-POW2-NEXT: store <7 x float> [[TMP7]], ptr [[GEP_4]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @test_insert_loads( +; POW2-ONLY-SAME: ptr [[A:%.*]], ptr noalias [[B:%.*]], float [[TMP0:%.*]]) #[[ATTR0:[0-9]+]] { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[MULADD_0:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 1.000000e+00, float 1.000000e+00) +; POW2-ONLY-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP3:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> , <2 x float> ) +; POW2-ONLY-NEXT: [[A_28:%.*]] = getelementptr i8, ptr [[A]], i64 28 +; POW2-ONLY-NEXT: [[L_A_28:%.*]] = load float, ptr [[A_28]], align 4 +; POW2-ONLY-NEXT: [[A_12:%.*]] = getelementptr i8, ptr [[A]], i64 12 +; POW2-ONLY-NEXT: [[L_A_12:%.*]] = load float, ptr [[A_12]], align 4 +; POW2-ONLY-NEXT: [[GEP_4:%.*]] = getelementptr i8, ptr [[B]], i64 4 +; POW2-ONLY-NEXT: [[L_B_0:%.*]] = load float, ptr [[B]], align 4 +; POW2-ONLY-NEXT: [[GEP_28:%.*]] = getelementptr i8, ptr [[B]], i64 28 +; POW2-ONLY-NEXT: [[GEP_20:%.*]] = getelementptr i8, ptr [[B]], i64 20 +; POW2-ONLY-NEXT: [[TMP4:%.*]] = insertelement <4 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP5:%.*]] = shufflevector <4 x float> [[TMP4]], <4 x float> poison, <4 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP6:%.*]] = insertelement <4 x float> , float [[L_A_12]], i32 0 +; POW2-ONLY-NEXT: [[TMP7:%.*]] = insertelement <4 x float> [[TMP6]], float [[L_A_28]], i32 1 +; POW2-ONLY-NEXT: [[TMP8:%.*]] = shufflevector <4 x float> [[TMP7]], <4 x float> poison, <4 x i32> +; POW2-ONLY-NEXT: [[TMP9:%.*]] = insertelement <4 x float> , float [[L_B_0]], i32 0 +; POW2-ONLY-NEXT: [[TMP10:%.*]] = call <4 x float> @llvm.fmuladd.v4f32(<4 x float> [[TMP5]], <4 x float> [[TMP8]], <4 x float> [[TMP9]]) +; POW2-ONLY-NEXT: store <4 x float> [[TMP10]], ptr [[GEP_4]], align 4 +; POW2-ONLY-NEXT: store <2 x float> [[TMP3]], ptr [[GEP_20]], align 4 +; POW2-ONLY-NEXT: store float [[MULADD_0]], ptr [[GEP_28]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %muladd.0 = tail call float @llvm.fmuladd.f32(float %0, float 1.000000e+00, float 1.000000e+00) diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll b/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll index 1faeea716b176..1399b4c35c781 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/vec3-reorder-reshuffle.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 -; RUN: opt -passes=slp-vectorizer -mtriple=x86_64-apple-macosx -S %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -mtriple=x86_64-apple-macosx -S %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s %struct.zot = type { i32, i32, i32 } @@ -138,21 +139,34 @@ if.end668: ; preds = %if.then665, %entry } define void @gather_2(ptr %mat1, float %0, float %1) { -; CHECK-LABEL: define void @gather_2( -; CHECK-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> , float [[TMP1]], i32 1 -; CHECK-NEXT: [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x float> zeroinitializer) -; CHECK-NEXT: [[TMP6:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00) -; CHECK-NEXT: [[TMP7:%.*]] = fmul float [[TMP6]], 0.000000e+00 -; CHECK-NEXT: [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1 -; CHECK-NEXT: [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2 -; CHECK-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer -; CHECK-NEXT: store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4 -; CHECK-NEXT: store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @gather_2( +; NON-POW2-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0 +; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP2]], <3 x float> poison, <3 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP5:%.*]] = insertelement <3 x float> , float [[TMP1]], i32 1 +; NON-POW2-NEXT: [[TMP6:%.*]] = shufflevector <3 x float> [[TMP5]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP4]], <3 x float> [[TMP6]], <3 x float> zeroinitializer) +; NON-POW2-NEXT: [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1 +; NON-POW2-NEXT: [[TMP8:%.*]] = fmul <3 x float> [[TMP7]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @gather_2( +; POW2-ONLY-SAME: ptr [[MAT1:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP4:%.*]] = insertelement <2 x float> , float [[TMP1]], i32 1 +; POW2-ONLY-NEXT: [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP3]], <2 x float> [[TMP4]], <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: [[TMP6:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP1]], float 0.000000e+00) +; POW2-ONLY-NEXT: [[TMP7:%.*]] = fmul float [[TMP6]], 0.000000e+00 +; POW2-ONLY-NEXT: [[ARRAYIDX163:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1 +; POW2-ONLY-NEXT: [[ARRAYIDX5_I_I_I280:%.*]] = getelementptr [4 x [4 x float]], ptr [[MAT1]], i64 0, i64 1, i64 2 +; POW2-ONLY-NEXT: [[TMP8:%.*]] = fmul <2 x float> [[TMP5]], zeroinitializer +; POW2-ONLY-NEXT: store <2 x float> [[TMP8]], ptr [[ARRAYIDX163]], align 4 +; POW2-ONLY-NEXT: store float [[TMP7]], ptr [[ARRAYIDX5_I_I_I280]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %2 = call float @llvm.fmuladd.f32(float %0, float 0.000000e+00, float 0.000000e+00) @@ -171,31 +185,47 @@ entry: } define i32 @reorder_indices_1(float %0) { -; CHECK-LABEL: define i32 @reorder_indices_1( -; CHECK-SAME: float [[TMP0:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4 -; CHECK-NEXT: [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]] -; CHECK-NEXT: [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]] -; CHECK-NEXT: [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]]) -; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> -; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1 -; CHECK-NEXT: [[TMP8:%.*]] = fneg <2 x float> [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]]) -; CHECK-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer) -; CHECK-NEXT: [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00) -; CHECK-NEXT: [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer -; CHECK-NEXT: [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00 -; CHECK-NEXT: store <2 x float> [[TMP15]], ptr [[NOR1]], align 4 -; CHECK-NEXT: store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4 -; CHECK-NEXT: ret i32 0 +; NON-POW2-LABEL: define i32 @reorder_indices_1( +; NON-POW2-SAME: float [[TMP0:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[NOR1]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP3:%.*]] = fneg <3 x float> [[TMP2]] +; NON-POW2-NEXT: [[TMP4:%.*]] = insertelement <3 x float> poison, float [[TMP0]], i32 0 +; NON-POW2-NEXT: [[TMP5:%.*]] = shufflevector <3 x float> [[TMP4]], <3 x float> poison, <3 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP6:%.*]] = fmul <3 x float> [[TMP3]], [[TMP5]] +; NON-POW2-NEXT: [[TMP7:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP1]], <3 x float> zeroinitializer, <3 x float> [[TMP6]]) +; NON-POW2-NEXT: [[TMP8:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP5]], <3 x float> [[TMP7]], <3 x float> zeroinitializer) +; NON-POW2-NEXT: [[TMP9:%.*]] = fmul <3 x float> [[TMP8]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP9]], ptr [[NOR1]], align 4 +; NON-POW2-NEXT: ret i32 0 +; +; POW2-ONLY-LABEL: define i32 @reorder_indices_1( +; POW2-ONLY-SAME: float [[TMP0:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[NOR1:%.*]] = alloca [0 x [3 x float]], i32 0, align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX2_I265:%.*]] = getelementptr float, ptr [[NOR1]], i64 2 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load float, ptr [[ARRAYIDX2_I265]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[NOR1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = extractelement <2 x float> [[TMP2]], i32 0 +; POW2-ONLY-NEXT: [[TMP4:%.*]] = fneg float [[TMP3]] +; POW2-ONLY-NEXT: [[NEG11_I:%.*]] = fmul float [[TMP4]], [[TMP0]] +; POW2-ONLY-NEXT: [[TMP5:%.*]] = call float @llvm.fmuladd.f32(float [[TMP1]], float 0.000000e+00, float [[NEG11_I]]) +; POW2-ONLY-NEXT: [[TMP6:%.*]] = shufflevector <2 x float> [[TMP2]], <2 x float> poison, <2 x i32> +; POW2-ONLY-NEXT: [[TMP7:%.*]] = insertelement <2 x float> [[TMP6]], float [[TMP1]], i32 1 +; POW2-ONLY-NEXT: [[TMP8:%.*]] = fneg <2 x float> [[TMP7]] +; POW2-ONLY-NEXT: [[TMP9:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP10:%.*]] = shufflevector <2 x float> [[TMP9]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP11:%.*]] = fmul <2 x float> [[TMP8]], [[TMP10]] +; POW2-ONLY-NEXT: [[TMP12:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> zeroinitializer, <2 x float> [[TMP11]]) +; POW2-ONLY-NEXT: [[TMP13:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP10]], <2 x float> [[TMP12]], <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: [[TMP14:%.*]] = call float @llvm.fmuladd.f32(float [[TMP0]], float [[TMP5]], float 0.000000e+00) +; POW2-ONLY-NEXT: [[TMP15:%.*]] = fmul <2 x float> [[TMP13]], zeroinitializer +; POW2-ONLY-NEXT: [[MUL6_I_I_I:%.*]] = fmul float [[TMP14]], 0.000000e+00 +; POW2-ONLY-NEXT: store <2 x float> [[TMP15]], ptr [[NOR1]], align 4 +; POW2-ONLY-NEXT: store float [[MUL6_I_I_I]], ptr [[ARRAYIDX2_I265]], align 4 +; POW2-ONLY-NEXT: ret i32 0 ; entry: %nor1 = alloca [0 x [3 x float]], i32 0, align 4 @@ -226,19 +256,28 @@ entry: } define void @reorder_indices_2(ptr %spoint) { -; CHECK-LABEL: define void @reorder_indices_2( -; CHECK-SAME: ptr [[SPOINT:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0 -; CHECK-NEXT: [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00) -; CHECK-NEXT: [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00 -; CHECK-NEXT: [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0 -; CHECK-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer) -; CHECK-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer -; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[DSCO]], align 4 -; CHECK-NEXT: [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2 -; CHECK-NEXT: store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @reorder_indices_2( +; NON-POW2-SAME: ptr [[SPOINT:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0 +; NON-POW2-NEXT: [[TMP0:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> zeroinitializer, <3 x float> zeroinitializer, <3 x float> zeroinitializer) +; NON-POW2-NEXT: [[TMP1:%.*]] = fmul <3 x float> [[TMP0]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP1]], ptr [[DSCO]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @reorder_indices_2( +; POW2-ONLY-SAME: ptr [[SPOINT:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP0:%.*]] = extractelement <3 x float> zeroinitializer, i64 0 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = tail call float @llvm.fmuladd.f32(float [[TMP0]], float 0.000000e+00, float 0.000000e+00) +; POW2-ONLY-NEXT: [[MUL4_I461:%.*]] = fmul float [[TMP1]], 0.000000e+00 +; POW2-ONLY-NEXT: [[DSCO:%.*]] = getelementptr float, ptr [[SPOINT]], i64 0 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> zeroinitializer, <2 x float> zeroinitializer, <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: [[TMP3:%.*]] = fmul <2 x float> [[TMP2]], zeroinitializer +; POW2-ONLY-NEXT: store <2 x float> [[TMP3]], ptr [[DSCO]], align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX5_I476:%.*]] = getelementptr float, ptr [[SPOINT]], i64 2 +; POW2-ONLY-NEXT: store float [[MUL4_I461]], ptr [[ARRAYIDX5_I476]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %0 = extractelement <3 x float> zeroinitializer, i64 1 @@ -290,19 +329,30 @@ entry: } define void @reuse_shuffle_indidces_1(ptr %col, float %0, float %1) { -; CHECK-LABEL: define void @reuse_shuffle_indidces_1( -; CHECK-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1 -; CHECK-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer -; CHECK-NEXT: store <2 x float> [[TMP5]], ptr [[COL]], align 4 -; CHECK-NEXT: [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2 -; CHECK-NEXT: [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00 -; CHECK-NEXT: [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00 -; CHECK-NEXT: store float [[TMP6]], ptr [[ARRAYIDX33]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @reuse_shuffle_indidces_1( +; NON-POW2-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[TMP2:%.*]] = insertelement <3 x float> poison, float [[TMP1]], i32 0 +; NON-POW2-NEXT: [[TMP3:%.*]] = insertelement <3 x float> [[TMP2]], float [[TMP0]], i32 1 +; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP5:%.*]] = fmul <3 x float> [[TMP4]], zeroinitializer +; NON-POW2-NEXT: [[TMP6:%.*]] = fadd <3 x float> [[TMP5]], zeroinitializer +; NON-POW2-NEXT: store <3 x float> [[TMP6]], ptr [[COL]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @reuse_shuffle_indidces_1( +; POW2-ONLY-SAME: ptr [[COL:%.*]], float [[TMP0:%.*]], float [[TMP1:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[TMP2:%.*]] = insertelement <2 x float> poison, float [[TMP1]], i32 0 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = insertelement <2 x float> [[TMP2]], float [[TMP0]], i32 1 +; POW2-ONLY-NEXT: [[TMP4:%.*]] = fmul <2 x float> [[TMP3]], zeroinitializer +; POW2-ONLY-NEXT: [[TMP5:%.*]] = fadd <2 x float> [[TMP4]], zeroinitializer +; POW2-ONLY-NEXT: store <2 x float> [[TMP5]], ptr [[COL]], align 4 +; POW2-ONLY-NEXT: [[ARRAYIDX33:%.*]] = getelementptr float, ptr [[COL]], i64 2 +; POW2-ONLY-NEXT: [[MUL38:%.*]] = fmul float [[TMP0]], 0.000000e+00 +; POW2-ONLY-NEXT: [[TMP6:%.*]] = fadd float [[MUL38]], 0.000000e+00 +; POW2-ONLY-NEXT: store float [[TMP6]], ptr [[ARRAYIDX33]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %mul24 = fmul float %1, 0.000000e+00 @@ -350,20 +400,32 @@ entry: } define void @reuse_shuffle_indices_cost_crash_2(ptr %bezt, float %0) { -; CHECK-LABEL: define void @reuse_shuffle_indices_cost_crash_2( -; CHECK-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00 -; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 -; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[FNEG]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> [[TMP4]], <2 x float> zeroinitializer) -; CHECK-NEXT: store <2 x float> [[TMP5]], ptr [[BEZT]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = tail call float @llvm.fmuladd.f32(float [[FNEG]], float 0.000000e+00, float 0.000000e+00) -; CHECK-NEXT: [[ARRAYIDX8_I831:%.*]] = getelementptr float, ptr [[BEZT]], i64 2 -; CHECK-NEXT: store float [[TMP6]], ptr [[ARRAYIDX8_I831]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @reuse_shuffle_indices_cost_crash_2( +; NON-POW2-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00 +; NON-POW2-NEXT: [[TMP1:%.*]] = insertelement <3 x float> poison, float [[FNEG]], i32 0 +; NON-POW2-NEXT: [[TMP2:%.*]] = shufflevector <3 x float> [[TMP1]], <3 x float> poison, <3 x i32> zeroinitializer +; NON-POW2-NEXT: [[TMP3:%.*]] = insertelement <3 x float> , float [[TMP0]], i32 0 +; NON-POW2-NEXT: [[TMP4:%.*]] = shufflevector <3 x float> [[TMP3]], <3 x float> poison, <3 x i32> +; NON-POW2-NEXT: [[TMP5:%.*]] = call <3 x float> @llvm.fmuladd.v3f32(<3 x float> [[TMP2]], <3 x float> [[TMP4]], <3 x float> zeroinitializer) +; NON-POW2-NEXT: store <3 x float> [[TMP5]], ptr [[BEZT]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @reuse_shuffle_indices_cost_crash_2( +; POW2-ONLY-SAME: ptr [[BEZT:%.*]], float [[TMP0:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[FNEG:%.*]] = fmul float [[TMP0]], 0.000000e+00 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP3:%.*]] = insertelement <2 x float> poison, float [[FNEG]], i32 0 +; POW2-ONLY-NEXT: [[TMP4:%.*]] = shufflevector <2 x float> [[TMP3]], <2 x float> poison, <2 x i32> zeroinitializer +; POW2-ONLY-NEXT: [[TMP5:%.*]] = call <2 x float> @llvm.fmuladd.v2f32(<2 x float> [[TMP2]], <2 x float> [[TMP4]], <2 x float> zeroinitializer) +; POW2-ONLY-NEXT: store <2 x float> [[TMP5]], ptr [[BEZT]], align 4 +; POW2-ONLY-NEXT: [[TMP6:%.*]] = tail call float @llvm.fmuladd.f32(float [[FNEG]], float 0.000000e+00, float 0.000000e+00) +; POW2-ONLY-NEXT: [[ARRAYIDX8_I831:%.*]] = getelementptr float, ptr [[BEZT]], i64 2 +; POW2-ONLY-NEXT: store float [[TMP6]], ptr [[ARRAYIDX8_I831]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %fneg = fmul float %0, 0.000000e+00 @@ -487,15 +549,21 @@ entry: } define void @vec3_extract(<3 x i16> %pixel.sroa.0.4.vec.insert606, ptr %call3.i536) { -; CHECK-LABEL: define void @vec3_extract( -; CHECK-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) { -; CHECK-NEXT: entry: -; CHECK-NEXT: [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2 -; CHECK-NEXT: [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2 -; CHECK-NEXT: store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2 -; CHECK-NEXT: [[TMP0:%.*]] = shufflevector <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], <3 x i16> poison, <2 x i32> -; CHECK-NEXT: store <2 x i16> [[TMP0]], ptr [[CALL3_I536]], align 2 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: define void @vec3_extract( +; NON-POW2-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) { +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: store <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], ptr [[CALL3_I536]], align 2 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: define void @vec3_extract( +; POW2-ONLY-SAME: <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606:%.*]], ptr [[CALL3_I536:%.*]]) { +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[PIXEL_SROA_0_4_VEC_EXTRACT:%.*]] = extractelement <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], i64 2 +; POW2-ONLY-NEXT: [[RED668:%.*]] = getelementptr i16, ptr [[CALL3_I536]], i64 2 +; POW2-ONLY-NEXT: store i16 [[PIXEL_SROA_0_4_VEC_EXTRACT]], ptr [[RED668]], align 2 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = shufflevector <3 x i16> [[PIXEL_SROA_0_4_VEC_INSERT606]], <3 x i16> poison, <2 x i32> +; POW2-ONLY-NEXT: store <2 x i16> [[TMP0]], ptr [[CALL3_I536]], align 2 +; POW2-ONLY-NEXT: ret void ; entry: %pixel.sroa.0.4.vec.extract = extractelement <3 x i16> %pixel.sroa.0.4.vec.insert606, i64 2 diff --git a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll index e8395fe69ea6b..e30cb76d53d92 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll @@ -1,12 +1,13 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -passes=slp-vectorizer -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2 -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck --check-prefixes=CHECK,NON-POW2 %s +; RUN: opt -passes=slp-vectorizer -slp-vectorize-non-power-of-2=false -S -mtriple=x86_64-unknown-linux-gnu -mcpu=bdver2 < %s | FileCheck --check-prefixes=CHECK,POW2-ONLY %s define void @add0(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @add0( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], -; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[TMP0]], +; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -32,21 +33,32 @@ entry: } define void @add1(ptr noalias %dst, ptr noalias %src) { -; CHECK-LABEL: @add1( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4 -; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1 -; CHECK-NEXT: store i32 [[TMP0]], ptr [[DST]], align 4 -; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3 -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], -; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4 -; CHECK-NEXT: [[ADD9:%.*]] = add nsw i32 [[TMP5]], 3 -; CHECK-NEXT: store i32 [[ADD9]], ptr [[INCDEC_PTR7]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @add1( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1 +; NON-POW2-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4 +; NON-POW2-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1 +; NON-POW2-NEXT: store i32 [[TMP0]], ptr [[DST]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[INCDEC_PTR]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = add nsw <3 x i32> [[TMP1]], +; NON-POW2-NEXT: store <3 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @add1( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1 +; POW2-ONLY-NEXT: store i32 [[TMP0]], ptr [[DST]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3 +; POW2-ONLY-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = add nsw <2 x i32> [[TMP1]], +; POW2-ONLY-NEXT: store <2 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4 +; POW2-ONLY-NEXT: [[ADD9:%.*]] = add nsw i32 [[TMP3]], 3 +; POW2-ONLY-NEXT: store i32 [[ADD9]], ptr [[INCDEC_PTR7]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %incdec.ptr = getelementptr inbounds i32, ptr %src, i64 1 @@ -81,9 +93,9 @@ define void @sub0(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[INCDEC_PTR]], align 4 ; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 2 ; CHECK-NEXT: store i32 [[TMP1]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR2]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = add nsw <2 x i32> [[TMP3]], -; CHECK-NEXT: store <2 x i32> [[TMP4]], ptr [[INCDEC_PTR3]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], +; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[INCDEC_PTR3]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -110,9 +122,9 @@ entry: define void @sub1(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @sub1( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], -; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[TMP0]], +; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -140,9 +152,9 @@ entry: define void @sub2(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @sub2( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], -; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add nsw <4 x i32> [[TMP0]], +; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -218,11 +230,11 @@ define void @addsub1(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> [[TMP2]], <2 x i32> ; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[DST]], align 4 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4 ; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3 -; CHECK-NEXT: store i32 [[TMP6]], ptr [[INCDEC_PTR3]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4 -; CHECK-NEXT: [[SUB8:%.*]] = sub nsw i32 [[TMP7]], -3 +; CHECK-NEXT: store i32 [[TMP4]], ptr [[INCDEC_PTR3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[SUB8:%.*]] = sub nsw i32 [[TMP5]], -3 ; CHECK-NEXT: store i32 [[SUB8]], ptr [[INCDEC_PTR6]], align 4 ; CHECK-NEXT: ret void ; @@ -252,15 +264,15 @@ define void @mul(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 2 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[SRC]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <2 x i32> [[TMP1]], -; CHECK-NEXT: store <2 x i32> [[TMP2]], ptr [[DST]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr [[SRC]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = mul nsw <2 x i32> [[TMP0]], +; CHECK-NEXT: store <2 x i32> [[TMP1]], ptr [[DST]], align 4 ; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[INCDEC_PTR2]], align 4 ; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3 -; CHECK-NEXT: store i32 [[TMP4]], ptr [[INCDEC_PTR4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4 -; CHECK-NEXT: [[MUL9:%.*]] = mul nsw i32 [[TMP5]], -9 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR5]], align 4 +; CHECK-NEXT: [[MUL9:%.*]] = mul nsw i32 [[TMP3]], -9 ; CHECK-NEXT: store i32 [[MUL9]], ptr [[INCDEC_PTR7]], align 4 ; CHECK-NEXT: ret void ; @@ -286,21 +298,32 @@ entry: } define void @shl0(ptr noalias %dst, ptr noalias %src) { -; CHECK-LABEL: @shl0( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4 -; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1 -; CHECK-NEXT: store i32 [[TMP0]], ptr [[DST]], align 4 -; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3 -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = shl <2 x i32> [[TMP2]], -; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4 -; CHECK-NEXT: [[SHL8:%.*]] = shl i32 [[TMP5]], 3 -; CHECK-NEXT: store i32 [[SHL8]], ptr [[INCDEC_PTR6]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @shl0( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1 +; NON-POW2-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4 +; NON-POW2-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1 +; NON-POW2-NEXT: store i32 [[TMP0]], ptr [[DST]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[INCDEC_PTR]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = shl <3 x i32> [[TMP1]], +; NON-POW2-NEXT: store <3 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @shl0( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 1 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load i32, ptr [[SRC]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 1 +; POW2-ONLY-NEXT: store i32 [[TMP0]], ptr [[DST]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, ptr [[SRC]], i64 3 +; POW2-ONLY-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, ptr [[DST]], i64 3 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[INCDEC_PTR]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = shl <2 x i32> [[TMP1]], +; POW2-ONLY-NEXT: store <2 x i32> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = load i32, ptr [[INCDEC_PTR4]], align 4 +; POW2-ONLY-NEXT: [[SHL8:%.*]] = shl i32 [[TMP3]], 3 +; POW2-ONLY-NEXT: store i32 [[SHL8]], ptr [[INCDEC_PTR6]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %incdec.ptr = getelementptr inbounds i32, ptr %src, i64 1 @@ -326,9 +349,9 @@ entry: define void @shl1(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @shl1( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = shl <4 x i32> [[TMP1]], -; CHECK-NEXT: store <4 x i32> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = shl <4 x i32> [[TMP0]], +; CHECK-NEXT: store <4 x i32> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -356,9 +379,9 @@ entry: define void @add0f(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @add0f( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], -; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fadd fast <4 x float> [[TMP0]], +; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -384,21 +407,32 @@ entry: } define void @add1f(ptr noalias %dst, ptr noalias %src) { -; CHECK-LABEL: @add1f( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4 -; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1 -; CHECK-NEXT: store float [[TMP0]], ptr [[DST]], align 4 -; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = fadd fast <2 x float> [[TMP2]], -; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 -; CHECK-NEXT: [[ADD9:%.*]] = fadd fast float [[TMP5]], 3.000000e+00 -; CHECK-NEXT: store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @add1f( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1 +; NON-POW2-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4 +; NON-POW2-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1 +; NON-POW2-NEXT: store float [[TMP0]], ptr [[DST]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[INCDEC_PTR]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = fadd fast <3 x float> [[TMP1]], +; NON-POW2-NEXT: store <3 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @add1f( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1 +; POW2-ONLY-NEXT: store float [[TMP0]], ptr [[DST]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 +; POW2-ONLY-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = fadd fast <2 x float> [[TMP1]], +; POW2-ONLY-NEXT: store <2 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 +; POW2-ONLY-NEXT: [[ADD9:%.*]] = fadd fast float [[TMP3]], 3.000000e+00 +; POW2-ONLY-NEXT: store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %incdec.ptr = getelementptr inbounds float, ptr %src, i64 1 @@ -433,9 +467,9 @@ define void @sub0f(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[INCDEC_PTR]], align 4 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2 ; CHECK-NEXT: store float [[TMP1]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = fadd fast <2 x float> [[TMP3]], -; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = fadd fast <2 x float> [[TMP2]], +; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[INCDEC_PTR4]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -462,9 +496,9 @@ entry: define void @sub1f(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @sub1f( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], -; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fadd fast <4 x float> [[TMP0]], +; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -492,9 +526,9 @@ entry: define void @sub2f(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @sub2f( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], -; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fadd fast <4 x float> [[TMP0]], +; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -570,11 +604,11 @@ define void @addsub1f(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x float> [[TMP1]], <2 x float> [[TMP2]], <2 x i32> ; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[DST]], align 4 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[TMP6:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4 ; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 -; CHECK-NEXT: store float [[TMP6]], ptr [[INCDEC_PTR3]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[INCDEC_PTR4]], align 4 -; CHECK-NEXT: [[SUB8:%.*]] = fsub fast float [[TMP7]], -3.000000e+00 +; CHECK-NEXT: store float [[TMP4]], ptr [[INCDEC_PTR3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[SUB8:%.*]] = fsub fast float [[TMP5]], -3.000000e+00 ; CHECK-NEXT: store float [[SUB8]], ptr [[INCDEC_PTR6]], align 4 ; CHECK-NEXT: ret void ; @@ -604,15 +638,15 @@ define void @mulf(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 2 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[SRC]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x float> [[TMP1]], -; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[DST]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[SRC]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fmul fast <2 x float> [[TMP0]], +; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[DST]], align 4 ; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4 ; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 -; CHECK-NEXT: store float [[TMP4]], ptr [[INCDEC_PTR4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 -; CHECK-NEXT: [[SUB9:%.*]] = fmul fast float [[TMP5]], -9.000000e+00 +; CHECK-NEXT: store float [[TMP2]], ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 +; CHECK-NEXT: [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00 ; CHECK-NEXT: store float [[SUB9]], ptr [[INCDEC_PTR7]], align 4 ; CHECK-NEXT: ret void ; @@ -640,9 +674,9 @@ entry: define void @add0fn(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @add0fn( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], -; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[TMP0]], +; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -668,21 +702,32 @@ entry: } define void @add1fn(ptr noalias %dst, ptr noalias %src) { -; CHECK-LABEL: @add1fn( -; CHECK-NEXT: entry: -; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1 -; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4 -; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1 -; CHECK-NEXT: store float [[TMP0]], ptr [[DST]], align 4 -; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 -; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x float> [[TMP2]], -; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 -; CHECK-NEXT: [[ADD9:%.*]] = fadd float [[TMP5]], 3.000000e+00 -; CHECK-NEXT: store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4 -; CHECK-NEXT: ret void +; NON-POW2-LABEL: @add1fn( +; NON-POW2-NEXT: entry: +; NON-POW2-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1 +; NON-POW2-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4 +; NON-POW2-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1 +; NON-POW2-NEXT: store float [[TMP0]], ptr [[DST]], align 4 +; NON-POW2-NEXT: [[TMP1:%.*]] = load <3 x float>, ptr [[INCDEC_PTR]], align 4 +; NON-POW2-NEXT: [[TMP2:%.*]] = fadd <3 x float> [[TMP1]], +; NON-POW2-NEXT: store <3 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; NON-POW2-NEXT: ret void +; +; POW2-ONLY-LABEL: @add1fn( +; POW2-ONLY-NEXT: entry: +; POW2-ONLY-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 1 +; POW2-ONLY-NEXT: [[TMP0:%.*]] = load float, ptr [[SRC]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 1 +; POW2-ONLY-NEXT: store float [[TMP0]], ptr [[DST]], align 4 +; POW2-ONLY-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 +; POW2-ONLY-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 +; POW2-ONLY-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[INCDEC_PTR]], align 4 +; POW2-ONLY-NEXT: [[TMP2:%.*]] = fadd <2 x float> [[TMP1]], +; POW2-ONLY-NEXT: store <2 x float> [[TMP2]], ptr [[INCDEC_PTR1]], align 4 +; POW2-ONLY-NEXT: [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 +; POW2-ONLY-NEXT: [[ADD9:%.*]] = fadd float [[TMP3]], 3.000000e+00 +; POW2-ONLY-NEXT: store float [[ADD9]], ptr [[INCDEC_PTR7]], align 4 +; POW2-ONLY-NEXT: ret void ; entry: %incdec.ptr = getelementptr inbounds float, ptr %src, i64 1 @@ -717,9 +762,9 @@ define void @sub0fn(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr [[INCDEC_PTR]], align 4 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 2 ; CHECK-NEXT: store float [[TMP1]], ptr [[INCDEC_PTR1]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = fadd <2 x float> [[TMP3]], -; CHECK-NEXT: store <2 x float> [[TMP4]], ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = fadd <2 x float> [[TMP2]], +; CHECK-NEXT: store <2 x float> [[TMP3]], ptr [[INCDEC_PTR4]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -746,9 +791,9 @@ entry: define void @sub1fn(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @sub1fn( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], -; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[TMP0]], +; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -776,9 +821,9 @@ entry: define void @sub2fn(ptr noalias %dst, ptr noalias %src) { ; CHECK-LABEL: @sub2fn( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> [[TMP1]], -; CHECK-NEXT: store <4 x float> [[TMP2]], ptr [[DST:%.*]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[SRC:%.*]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fadd <4 x float> [[TMP0]], +; CHECK-NEXT: store <4 x float> [[TMP1]], ptr [[DST:%.*]], align 4 ; CHECK-NEXT: ret void ; entry: @@ -808,15 +853,15 @@ define void @mulfn(ptr noalias %dst, ptr noalias %src) { ; CHECK-NEXT: entry: ; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, ptr [[SRC:%.*]], i64 2 ; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, ptr [[DST:%.*]], i64 2 -; CHECK-NEXT: [[TMP1:%.*]] = load <2 x float>, ptr [[SRC]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x float> [[TMP1]], -; CHECK-NEXT: store <2 x float> [[TMP2]], ptr [[DST]], align 4 +; CHECK-NEXT: [[TMP0:%.*]] = load <2 x float>, ptr [[SRC]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = fmul <2 x float> [[TMP0]], +; CHECK-NEXT: store <2 x float> [[TMP1]], ptr [[DST]], align 4 ; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, ptr [[SRC]], i64 3 -; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr [[INCDEC_PTR2]], align 4 ; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, ptr [[DST]], i64 3 -; CHECK-NEXT: store float [[TMP4]], ptr [[INCDEC_PTR4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 -; CHECK-NEXT: [[SUB9:%.*]] = fmul fast float [[TMP5]], -9.000000e+00 +; CHECK-NEXT: store float [[TMP2]], ptr [[INCDEC_PTR4]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = load float, ptr [[INCDEC_PTR5]], align 4 +; CHECK-NEXT: [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00 ; CHECK-NEXT: store float [[SUB9]], ptr [[INCDEC_PTR7]], align 4 ; CHECK-NEXT: ret void ;