diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 8508086ea3a4e..c11d558a73e9d 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -935,28 +935,31 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { ArrayRef Mask, VectorType *Ty, int &Index, VectorType *&SubTy) const { - if (Mask.empty()) + int Limit = Mask.size() * 2; + if (Mask.empty() || + // Extra check required by isSingleSourceMaskImpl function (called by + // ShuffleVectorInst::isSingleSourceMask). + any_of(Mask, [Limit](int I) { return I >= Limit; })) return Kind; - int NumSrcElts = Ty->getElementCount().getKnownMinValue(); switch (Kind) { case TTI::SK_PermuteSingleSrc: - if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts)) + if (ShuffleVectorInst::isReverseMask(Mask)) return TTI::SK_Reverse; - if (ShuffleVectorInst::isZeroEltSplatMask(Mask, NumSrcElts)) + if (ShuffleVectorInst::isZeroEltSplatMask(Mask)) return TTI::SK_Broadcast; break; case TTI::SK_PermuteTwoSrc: { int NumSubElts; if (Mask.size() > 2 && ShuffleVectorInst::isInsertSubvectorMask( - Mask, NumSrcElts, NumSubElts, Index)) { + Mask, Mask.size(), NumSubElts, Index)) { SubTy = FixedVectorType::get(Ty->getElementType(), NumSubElts); return TTI::SK_InsertSubvector; } - if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts)) + if (ShuffleVectorInst::isSelectMask(Mask)) return TTI::SK_Select; - if (ShuffleVectorInst::isTransposeMask(Mask, NumSrcElts)) + if (ShuffleVectorInst::isTransposeMask(Mask)) return TTI::SK_Transpose; - if (ShuffleVectorInst::isSpliceMask(Mask, NumSrcElts, Index)) + if (ShuffleVectorInst::isSpliceMask(Mask, Index)) return TTI::SK_Splice; break; } diff --git a/llvm/include/llvm/IR/Instructions.h b/llvm/include/llvm/IR/Instructions.h index c85727ce30a94..a0c8406fe4ec1 100644 --- a/llvm/include/llvm/IR/Instructions.h +++ b/llvm/include/llvm/IR/Instructions.h @@ -2068,14 +2068,13 @@ class ShuffleVectorInst : public Instruction { /// Return true if this shuffle mask chooses elements from exactly one source /// vector. /// Example: <7,5,undef,7> - /// This assumes that vector operands (of length \p NumSrcElts) are the same - /// length as the mask. - static bool isSingleSourceMask(ArrayRef Mask, int NumSrcElts); - static bool isSingleSourceMask(const Constant *Mask, int NumSrcElts) { + /// This assumes that vector operands are the same length as the mask. + static bool isSingleSourceMask(ArrayRef Mask); + static bool isSingleSourceMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isSingleSourceMask(MaskAsInts, NumSrcElts); + return isSingleSourceMask(MaskAsInts); } /// Return true if this shuffle chooses elements from exactly one source @@ -2083,8 +2082,7 @@ class ShuffleVectorInst : public Instruction { /// Example: shufflevector <4 x n> A, <4 x n> B, <3,0,undef,3> /// TODO: Optionally allow length-changing shuffles. bool isSingleSource() const { - return !changesLength() && - isSingleSourceMask(ShuffleMask, ShuffleMask.size()); + return !changesLength() && isSingleSourceMask(ShuffleMask); } /// Return true if this shuffle mask chooses elements from exactly one source @@ -2092,8 +2090,8 @@ class ShuffleVectorInst : public Instruction { /// necessarily a no-op because it may change the number of elements from its /// input vectors or it may provide demanded bits knowledge via undef lanes. /// Example: - static bool isIdentityMask(ArrayRef Mask, int NumSrcElts); - static bool isIdentityMask(const Constant *Mask, int NumSrcElts) { + static bool isIdentityMask(ArrayRef Mask); + static bool isIdentityMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); // Not possible to express a shuffle mask for a scalable vector for this @@ -2103,7 +2101,7 @@ class ShuffleVectorInst : public Instruction { SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isIdentityMask(MaskAsInts, NumSrcElts); + return isIdentityMask(MaskAsInts); } /// Return true if this shuffle chooses elements from exactly one source @@ -2116,7 +2114,7 @@ class ShuffleVectorInst : public Instruction { if (isa(getType())) return false; - return !changesLength() && isIdentityMask(ShuffleMask, ShuffleMask.size()); + return !changesLength() && isIdentityMask(ShuffleMask); } /// Return true if this shuffle lengthens exactly one source vector with @@ -2140,12 +2138,12 @@ class ShuffleVectorInst : public Instruction { /// In that case, the shuffle is better classified as an identity shuffle. /// This assumes that vector operands are the same length as the mask /// (a length-changing shuffle can never be equivalent to a vector select). - static bool isSelectMask(ArrayRef Mask, int NumSrcElts); - static bool isSelectMask(const Constant *Mask, int NumSrcElts) { + static bool isSelectMask(ArrayRef Mask); + static bool isSelectMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isSelectMask(MaskAsInts, NumSrcElts); + return isSelectMask(MaskAsInts); } /// Return true if this shuffle chooses elements from its source vectors @@ -2157,20 +2155,19 @@ class ShuffleVectorInst : public Instruction { /// In that case, the shuffle is better classified as an identity shuffle. /// TODO: Optionally allow length-changing shuffles. bool isSelect() const { - return !changesLength() && isSelectMask(ShuffleMask, ShuffleMask.size()); + return !changesLength() && isSelectMask(ShuffleMask); } /// Return true if this shuffle mask swaps the order of elements from exactly /// one source vector. /// Example: <7,6,undef,4> - /// This assumes that vector operands (of length \p NumSrcElts) are the same - /// length as the mask. - static bool isReverseMask(ArrayRef Mask, int NumSrcElts); - static bool isReverseMask(const Constant *Mask, int NumSrcElts) { + /// This assumes that vector operands are the same length as the mask. + static bool isReverseMask(ArrayRef Mask); + static bool isReverseMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isReverseMask(MaskAsInts, NumSrcElts); + return isReverseMask(MaskAsInts); } /// Return true if this shuffle swaps the order of elements from exactly @@ -2178,20 +2175,19 @@ class ShuffleVectorInst : public Instruction { /// Example: shufflevector <4 x n> A, <4 x n> B, <3,undef,1,undef> /// TODO: Optionally allow length-changing shuffles. bool isReverse() const { - return !changesLength() && isReverseMask(ShuffleMask, ShuffleMask.size()); + return !changesLength() && isReverseMask(ShuffleMask); } /// Return true if this shuffle mask chooses all elements with the same value /// as the first element of exactly one source vector. /// Example: <4,undef,undef,4> - /// This assumes that vector operands (of length \p NumSrcElts) are the same - /// length as the mask. - static bool isZeroEltSplatMask(ArrayRef Mask, int NumSrcElts); - static bool isZeroEltSplatMask(const Constant *Mask, int NumSrcElts) { + /// This assumes that vector operands are the same length as the mask. + static bool isZeroEltSplatMask(ArrayRef Mask); + static bool isZeroEltSplatMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isZeroEltSplatMask(MaskAsInts, NumSrcElts); + return isZeroEltSplatMask(MaskAsInts); } /// Return true if all elements of this shuffle are the same value as the @@ -2201,8 +2197,7 @@ class ShuffleVectorInst : public Instruction { /// TODO: Optionally allow length-changing shuffles. /// TODO: Optionally allow splats from other elements. bool isZeroEltSplat() const { - return !changesLength() && - isZeroEltSplatMask(ShuffleMask, ShuffleMask.size()); + return !changesLength() && isZeroEltSplatMask(ShuffleMask); } /// Return true if this shuffle mask is a transpose mask. @@ -2237,12 +2232,12 @@ class ShuffleVectorInst : public Instruction { /// ; Transposed matrix /// t0 = < a, e, c, g > = shufflevector m0, m1 < 0, 4, 2, 6 > /// t1 = < b, f, d, h > = shufflevector m0, m1 < 1, 5, 3, 7 > - static bool isTransposeMask(ArrayRef Mask, int NumSrcElts); - static bool isTransposeMask(const Constant *Mask, int NumSrcElts) { + static bool isTransposeMask(ArrayRef Mask); + static bool isTransposeMask(const Constant *Mask) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isTransposeMask(MaskAsInts, NumSrcElts); + return isTransposeMask(MaskAsInts); } /// Return true if this shuffle transposes the elements of its inputs without @@ -2251,21 +2246,19 @@ class ShuffleVectorInst : public Instruction { /// exact specification. /// Example: shufflevector <4 x n> A, <4 x n> B, <0,4,2,6> bool isTranspose() const { - return !changesLength() && isTransposeMask(ShuffleMask, ShuffleMask.size()); + return !changesLength() && isTransposeMask(ShuffleMask); } /// Return true if this shuffle mask is a splice mask, concatenating the two /// inputs together and then extracts an original width vector starting from /// the splice index. /// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4> - /// This assumes that vector operands (of length \p NumSrcElts) are the same - /// length as the mask. - static bool isSpliceMask(ArrayRef Mask, int NumSrcElts, int &Index); - static bool isSpliceMask(const Constant *Mask, int NumSrcElts, int &Index) { + static bool isSpliceMask(ArrayRef Mask, int &Index); + static bool isSpliceMask(const Constant *Mask, int &Index) { assert(Mask->getType()->isVectorTy() && "Shuffle needs vector constant."); SmallVector MaskAsInts; getShuffleMask(Mask, MaskAsInts); - return isSpliceMask(MaskAsInts, NumSrcElts, Index); + return isSpliceMask(MaskAsInts, Index); } /// Return true if this shuffle splices two inputs without changing the length @@ -2273,8 +2266,7 @@ class ShuffleVectorInst : public Instruction { /// then extracts an original width vector starting from the splice index. /// Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4> bool isSplice(int &Index) const { - return !changesLength() && - isSpliceMask(ShuffleMask, ShuffleMask.size(), Index); + return !changesLength() && isSpliceMask(ShuffleMask, Index); } /// Return true if this shuffle mask is an extract subvector mask. diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index f2bd5137bd209..05a19ed989bef 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -24151,7 +24151,7 @@ static SDValue foldExtractSubvectorFromShuffleVector(SDNode *N, // Profitability check: only deal with extractions from the first subvector // unless the mask becomes an identity mask. - if (!ShuffleVectorInst::isIdentityMask(NewMask, NewMask.size()) || + if (!ShuffleVectorInst::isIdentityMask(NewMask) || any_of(NewMask, [](int M) { return M < 0; })) for (auto &DemandedSubvector : DemandedSubvectors) if (DemandedSubvector.second != 0) diff --git a/llvm/lib/IR/Instructions.cpp b/llvm/lib/IR/Instructions.cpp index ece3b58792dd1..b76e355996ea5 100644 --- a/llvm/lib/IR/Instructions.cpp +++ b/llvm/lib/IR/Instructions.cpp @@ -2136,10 +2136,10 @@ static bool isSingleSourceMaskImpl(ArrayRef Mask, int NumOpElts) { return UsesLHS || UsesRHS; } -bool ShuffleVectorInst::isSingleSourceMask(ArrayRef Mask, int NumSrcElts) { +bool ShuffleVectorInst::isSingleSourceMask(ArrayRef Mask) { // We don't have vector operand size information, so assume operands are the // same size as the mask. - return isSingleSourceMaskImpl(Mask, NumSrcElts); + return isSingleSourceMaskImpl(Mask, Mask.size()); } static bool isIdentityMaskImpl(ArrayRef Mask, int NumOpElts) { @@ -2154,75 +2154,65 @@ static bool isIdentityMaskImpl(ArrayRef Mask, int NumOpElts) { return true; } -bool ShuffleVectorInst::isIdentityMask(ArrayRef Mask, int NumSrcElts) { - if (Mask.size() != static_cast(NumSrcElts)) - return false; +bool ShuffleVectorInst::isIdentityMask(ArrayRef Mask) { // We don't have vector operand size information, so assume operands are the // same size as the mask. - return isIdentityMaskImpl(Mask, NumSrcElts); + return isIdentityMaskImpl(Mask, Mask.size()); } -bool ShuffleVectorInst::isReverseMask(ArrayRef Mask, int NumSrcElts) { - if (Mask.size() != static_cast(NumSrcElts)) - return false; - if (!isSingleSourceMask(Mask, NumSrcElts)) +bool ShuffleVectorInst::isReverseMask(ArrayRef Mask) { + if (!isSingleSourceMask(Mask)) return false; // The number of elements in the mask must be at least 2. - if (NumSrcElts < 2) + int NumElts = Mask.size(); + if (NumElts < 2) return false; - for (int I = 0, E = Mask.size(); I < E; ++I) { - if (Mask[I] == -1) + for (int i = 0; i < NumElts; ++i) { + if (Mask[i] == -1) continue; - if (Mask[I] != (NumSrcElts - 1 - I) && - Mask[I] != (NumSrcElts + NumSrcElts - 1 - I)) + if (Mask[i] != (NumElts - 1 - i) && Mask[i] != (NumElts + NumElts - 1 - i)) return false; } return true; } -bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef Mask, int NumSrcElts) { - if (Mask.size() != static_cast(NumSrcElts)) - return false; - if (!isSingleSourceMask(Mask, NumSrcElts)) +bool ShuffleVectorInst::isZeroEltSplatMask(ArrayRef Mask) { + if (!isSingleSourceMask(Mask)) return false; - for (int I = 0, E = Mask.size(); I < E; ++I) { - if (Mask[I] == -1) + for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) { + if (Mask[i] == -1) continue; - if (Mask[I] != 0 && Mask[I] != NumSrcElts) + if (Mask[i] != 0 && Mask[i] != NumElts) return false; } return true; } -bool ShuffleVectorInst::isSelectMask(ArrayRef Mask, int NumSrcElts) { - if (Mask.size() != static_cast(NumSrcElts)) - return false; +bool ShuffleVectorInst::isSelectMask(ArrayRef Mask) { // Select is differentiated from identity. It requires using both sources. - if (isSingleSourceMask(Mask, NumSrcElts)) + if (isSingleSourceMask(Mask)) return false; - for (int I = 0, E = Mask.size(); I < E; ++I) { - if (Mask[I] == -1) + for (int i = 0, NumElts = Mask.size(); i < NumElts; ++i) { + if (Mask[i] == -1) continue; - if (Mask[I] != I && Mask[I] != (NumSrcElts + I)) + if (Mask[i] != i && Mask[i] != (NumElts + i)) return false; } return true; } -bool ShuffleVectorInst::isTransposeMask(ArrayRef Mask, int NumSrcElts) { +bool ShuffleVectorInst::isTransposeMask(ArrayRef Mask) { // Example masks that will return true: // v1 = // v2 = // trn1 = shufflevector v1, v2 <0, 4, 2, 6> = // trn2 = shufflevector v1, v2 <1, 5, 3, 7> = - if (Mask.size() != static_cast(NumSrcElts)) - return false; // 1. The number of elements in the mask must be a power-of-2 and at least 2. - int Sz = Mask.size(); - if (Sz < 2 || !isPowerOf2_32(Sz)) + int NumElts = Mask.size(); + if (NumElts < 2 || !isPowerOf2_32(NumElts)) return false; // 2. The first element of the mask must be either a 0 or a 1. @@ -2231,26 +2221,23 @@ bool ShuffleVectorInst::isTransposeMask(ArrayRef Mask, int NumSrcElts) { // 3. The difference between the first 2 elements must be equal to the // number of elements in the mask. - if ((Mask[1] - Mask[0]) != NumSrcElts) + if ((Mask[1] - Mask[0]) != NumElts) return false; // 4. The difference between consecutive even-numbered and odd-numbered // elements must be equal to 2. - for (int I = 2; I < Sz; ++I) { - int MaskEltVal = Mask[I]; + for (int i = 2; i < NumElts; ++i) { + int MaskEltVal = Mask[i]; if (MaskEltVal == -1) return false; - int MaskEltPrevVal = Mask[I - 2]; + int MaskEltPrevVal = Mask[i - 2]; if (MaskEltVal - MaskEltPrevVal != 2) return false; } return true; } -bool ShuffleVectorInst::isSpliceMask(ArrayRef Mask, int NumSrcElts, - int &Index) { - if (Mask.size() != static_cast(NumSrcElts)) - return false; +bool ShuffleVectorInst::isSpliceMask(ArrayRef Mask, int &Index) { // Example: shufflevector <4 x n> A, <4 x n> B, <1,2,3,4> int StartIndex = -1; for (int I = 0, E = Mask.size(); I != E; ++I) { @@ -2261,7 +2248,7 @@ bool ShuffleVectorInst::isSpliceMask(ArrayRef Mask, int NumSrcElts, if (StartIndex == -1) { // Don't support a StartIndex that begins in the second input, or if the // first non-undef index would access below the StartIndex. - if (MaskEltVal < I || NumSrcElts <= (MaskEltVal - I)) + if (MaskEltVal < I || E <= (MaskEltVal - I)) return false; StartIndex = MaskEltVal - I; @@ -2556,7 +2543,7 @@ bool ShuffleVectorInst::isOneUseSingleSourceMask(int VF) const { // case. if (isa(getType())) return false; - if (!isSingleSourceMask(ShuffleMask, VF)) + if (!isSingleSourceMask(ShuffleMask)) return false; return isOneUseSingleSourceMask(ShuffleMask, VF); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 3ae7a893ca4e9..5848667728878 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -11813,7 +11813,7 @@ SDValue AArch64TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, if (((VT.getVectorNumElements() == 8 && VT.getScalarSizeInBits() == 16) || (VT.getVectorNumElements() == 16 && VT.getScalarSizeInBits() == 8)) && - ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size())) { + ShuffleVectorInst::isReverseMask(ShuffleMask)) { SDValue Rev = DAG.getNode(AArch64ISD::REV64, dl, VT, V1); return DAG.getNode(AArch64ISD::EXT, dl, VT, Rev, Rev, DAG.getConstant(8, dl, MVT::i32)); @@ -25837,8 +25837,7 @@ static SDValue GenerateFixedLengthSVETBL(SDValue Op, SDValue Op1, SDValue Op2, SDLoc DL(Op); unsigned MinSVESize = Subtarget.getMinSVEVectorSizeInBits(); unsigned MaxSVESize = Subtarget.getMaxSVEVectorSizeInBits(); - bool IsSingleOp = - ShuffleVectorInst::isSingleSourceMask(ShuffleMask, ShuffleMask.size()); + bool IsSingleOp = ShuffleVectorInst::isSingleSourceMask(ShuffleMask); // Ignore two operands if no SVE2 or all index numbers couldn't // be represented. @@ -26022,8 +26021,7 @@ SDValue AArch64TargetLowering::LowerFixedLengthVECTOR_SHUFFLEToSVE( unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits(); unsigned MaxSVESize = Subtarget->getMaxSVEVectorSizeInBits(); if (MinSVESize == MaxSVESize && MaxSVESize == VT.getSizeInBits()) { - if (ShuffleVectorInst::isReverseMask(ShuffleMask, ShuffleMask.size()) && - Op2.isUndef()) { + if (ShuffleVectorInst::isReverseMask(ShuffleMask) && Op2.isUndef()) { Op = DAG.getNode(ISD::VECTOR_REVERSE, DL, ContainerVT, Op1); return convertFromScalableVector(DAG, VT, Op); } diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 8cca167e32ee7..52ce370bbacc6 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -8375,7 +8375,7 @@ bool ARMTargetLowering::isShuffleMaskLegal(ArrayRef M, EVT VT) const { unsigned EltSize = VT.getScalarSizeInBits(); if (EltSize >= 32 || ShuffleVectorSDNode::isSplatMask(&M[0], VT) || - ShuffleVectorInst::isIdentityMask(M, M.size()) || + ShuffleVectorInst::isIdentityMask(M) || isVREVMask(M, VT, 64) || isVREVMask(M, VT, 32) || isVREVMask(M, VT, 16)) diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 937ce78507028..413af1ff4b943 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -4339,8 +4339,7 @@ static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN, assert(VT.getVectorElementType() == MVT::i1); - if (!ShuffleVectorInst::isReverseMask(SVN->getMask(), - SVN->getMask().size()) || + if (!ShuffleVectorInst::isReverseMask(SVN->getMask()) || !SVN->getOperand(1).isUndef()) return SDValue(); diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index e947d4d1e8acd..2f510a215d80b 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -322,10 +322,7 @@ InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, LT.second.getVectorElementType().getSizeInBits() == Tp->getElementType()->getPrimitiveSizeInBits() && LT.second.getVectorNumElements() < - cast(Tp)->getNumElements() && - divideCeil(Mask.size(), - cast(Tp)->getNumElements()) == - static_cast(*LT.first.getValue())) { + cast(Tp)->getNumElements()) { unsigned NumRegs = *LT.first.getValue(); unsigned VF = cast(Tp)->getNumElements(); unsigned SubVF = PowerOf2Ceil(VF / NumRegs); @@ -501,7 +498,7 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost( InstructionCost Cost = MemCost; for (unsigned Index : Indices) { FixedVectorType *SubVecTy = - FixedVectorType::get(FVTy->getElementType(), VF * Factor); + FixedVectorType::get(FVTy->getElementType(), VF); auto Mask = createStrideMask(Index, Factor, VF); InstructionCost ShuffleCost = getShuffleCost(TTI::ShuffleKind::SK_PermuteSingleSrc, SubVecTy, Mask, diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 5a6413653908e..d838d1f96e310 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -1596,6 +1596,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, BaseTp->getElementType()->getPrimitiveSizeInBits() && LegalVT.getVectorNumElements() < cast(BaseTp)->getNumElements()) { + unsigned VecTySize = DL.getTypeStoreSize(BaseTp); unsigned LegalVTSize = LegalVT.getStoreSize(); // Number of source vectors after legalization: @@ -1620,10 +1621,6 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, // copy of the previous destination register (the cost is // TTI::TCC_Basic). If the source register is just reused, the cost for // this operation is 0. - NumOfDests = - getTypeLegalizationCost( - FixedVectorType::get(BaseTp->getElementType(), Mask.size())) - .first; unsigned E = *NumOfDests.getValue(); unsigned NormalizedVF = LegalVT.getVectorNumElements() * std::max(NumOfSrcs, E); @@ -1638,7 +1635,7 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, NormalizedMask, NumOfSrcRegs, NumOfDestRegs, NumOfDestRegs, []() {}, [this, SingleOpTy, CostKind, &PrevSrcReg, &PrevRegMask, &Cost](ArrayRef RegMask, unsigned SrcReg, unsigned DestReg) { - if (!ShuffleVectorInst::isIdentityMask(RegMask, RegMask.size())) { + if (!ShuffleVectorInst::isIdentityMask(RegMask)) { // Check if the previous register can be just copied to the next // one. if (PrevRegMask.empty() || PrevSrcReg != SrcReg || diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c8b58c51d4e6e..69d7593d5fc1e 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -2122,8 +2122,8 @@ static Instruction *foldSelectShuffleOfSelectShuffle(ShuffleVectorInst &Shuf) { NewMask[i] = Mask[i] < (signed)NumElts ? Mask[i] : Mask1[i]; // A select mask with undef elements might look like an identity mask. - assert((ShuffleVectorInst::isSelectMask(NewMask, NumElts) || - ShuffleVectorInst::isIdentityMask(NewMask, NumElts)) && + assert((ShuffleVectorInst::isSelectMask(NewMask) || + ShuffleVectorInst::isIdentityMask(NewMask)) && "Unexpected shuffle mask"); return new ShuffleVectorInst(X, Y, NewMask); } diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 70fcfd4cbde99..bd67b69e0b55d 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3812,7 +3812,7 @@ static void reorderOrder(SmallVectorImpl &Order, ArrayRef Mask) { inversePermutation(Order, MaskOrder); } reorderReuses(MaskOrder, Mask); - if (ShuffleVectorInst::isIdentityMask(MaskOrder, MaskOrder.size())) { + if (ShuffleVectorInst::isIdentityMask(MaskOrder)) { Order.clear(); return; } @@ -4333,7 +4333,7 @@ BoUpSLP::getReorderingData(const TreeEntry &TE, bool TopToBottom) { static bool isRepeatedNonIdentityClusteredMask(ArrayRef Mask, unsigned Sz) { ArrayRef FirstCluster = Mask.slice(0, Sz); - if (ShuffleVectorInst::isIdentityMask(FirstCluster, Sz)) + if (ShuffleVectorInst::isIdentityMask(FirstCluster)) return false; for (unsigned I = Sz, E = Mask.size(); I < E; I += Sz) { ArrayRef Cluster = Mask.slice(I, Sz); @@ -6646,24 +6646,9 @@ class BaseShuffleAnalysis { bool IsStrict) { int Limit = Mask.size(); int VF = VecTy->getNumElements(); - int Index = -1; - if (VF == Limit && ShuffleVectorInst::isIdentityMask(Mask, Limit)) - return true; - if (!IsStrict) { - // Consider extract subvector starting from index 0. - if (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) && - Index == 0) - return true; - // All VF-size submasks are identity (e.g. - // etc. for VF 4). - if (Limit % VF == 0 && all_of(seq(0, Limit / VF), [=](int Idx) { - ArrayRef Slice = Mask.slice(Idx * VF, VF); - return all_of(Slice, [](int I) { return I == PoisonMaskElem; }) || - ShuffleVectorInst::isIdentityMask(Slice, VF); - })) - return true; - } - return false; + return (VF == Limit || !IsStrict) && + all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) && + ShuffleVectorInst::isIdentityMask(Mask); } /// Tries to combine 2 different masks into single one. @@ -6733,8 +6718,7 @@ class BaseShuffleAnalysis { if (isIdentityMask(Mask, SVTy, /*IsStrict=*/false)) { if (!IdentityOp || !SinglePermute || (isIdentityMask(Mask, SVTy, /*IsStrict=*/true) && - !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask, - IdentityMask.size()))) { + !ShuffleVectorInst::isZeroEltSplatMask(IdentityMask))) { IdentityOp = SV; // Store current mask in the IdentityMask so later we did not lost // this info if IdentityOp is selected as the best candidate for the @@ -6804,7 +6788,7 @@ class BaseShuffleAnalysis { } if (auto *OpTy = dyn_cast(Op->getType()); !OpTy || !isIdentityMask(Mask, OpTy, SinglePermute) || - ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size())) { + ShuffleVectorInst::isZeroEltSplatMask(Mask)) { if (IdentityOp) { V = IdentityOp; assert(Mask.size() == IdentityMask.size() && @@ -6820,7 +6804,7 @@ class BaseShuffleAnalysis { /*IsStrict=*/true) || (Shuffle && Mask.size() == Shuffle->getShuffleMask().size() && Shuffle->isZeroEltSplat() && - ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size()))); + ShuffleVectorInst::isZeroEltSplatMask(Mask))); } V = Op; return false; @@ -6925,9 +6909,11 @@ class BaseShuffleAnalysis { CombinedMask1[I] = CombinedMask2[I] + (Op1 == Op2 ? 0 : VF); } } - if (Op1 == Op2 && - (ShuffleVectorInst::isIdentityMask(CombinedMask1, VF) || - (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1, VF) && + const int Limit = CombinedMask1.size() * 2; + if (Op1 == Op2 && Limit == 2 * VF && + all_of(CombinedMask1, [=](int Idx) { return Idx < Limit; }) && + (ShuffleVectorInst::isIdentityMask(CombinedMask1) || + (ShuffleVectorInst::isZeroEltSplatMask(CombinedMask1) && isa(Op1) && cast(Op1)->getShuffleMask() == ArrayRef(CombinedMask1)))) @@ -6967,20 +6953,6 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { SmallPtrSetImpl &CheckedExtracts; constexpr static TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; - static Constant *getAllOnesValue(const DataLayout &DL, Type *Ty) { - if (Ty->getScalarType()->isPointerTy()) { - Constant *Res = ConstantExpr::getIntToPtr( - ConstantInt::getAllOnesValue( - IntegerType::get(Ty->getContext(), - DL.getTypeStoreSizeInBits(Ty->getScalarType()))), - Ty->getScalarType()); - if (auto *VTy = dyn_cast(Ty)) - Res = ConstantVector::getSplat(VTy->getElementCount(), Res); - return Res; - } - return Constant::getAllOnesValue(Ty); - } - InstructionCost getBuildVectorCost(ArrayRef VL, Value *Root) { if ((!Root && allConstant(VL)) || all_of(VL, UndefValue::classof)) return TTI::TCC_Free; @@ -7211,12 +7183,11 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { const TargetTransformInfo &TTI; static bool isEmptyOrIdentity(ArrayRef Mask, unsigned VF) { - int Index = -1; + int Limit = 2 * VF; return Mask.empty() || (VF == Mask.size() && - ShuffleVectorInst::isIdentityMask(Mask, VF)) || - (ShuffleVectorInst::isExtractSubvectorMask(Mask, VF, Index) && - Index == 0); + all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) && + ShuffleVectorInst::isIdentityMask(Mask)); } public: @@ -7229,17 +7200,21 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { cast(V1->getType())->getElementCount().getKnownMinValue(); if (isEmptyOrIdentity(Mask, VF)) return TTI::TCC_Free; - return TTI.getShuffleCost(TTI::SK_PermuteTwoSrc, - cast(V1->getType()), Mask); + return TTI.getShuffleCost( + TTI::SK_PermuteTwoSrc, + FixedVectorType::get( + cast(V1->getType())->getElementType(), Mask.size()), + Mask); } InstructionCost createShuffleVector(Value *V1, ArrayRef Mask) const { // Empty mask or identity mask are free. - unsigned VF = - cast(V1->getType())->getElementCount().getKnownMinValue(); - if (isEmptyOrIdentity(Mask, VF)) + if (isEmptyOrIdentity(Mask, Mask.size())) return TTI::TCC_Free; - return TTI.getShuffleCost(TTI::SK_PermuteSingleSrc, - cast(V1->getType()), Mask); + return TTI.getShuffleCost( + TTI::SK_PermuteSingleSrc, + FixedVectorType::get( + cast(V1->getType())->getElementType(), Mask.size()), + Mask); } InstructionCost createIdentity(Value *) const { return TTI::TCC_Free; } InstructionCost createPoison(Type *Ty, unsigned VF) const { @@ -7256,119 +7231,47 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { const PointerUnion &P2, ArrayRef Mask) { ShuffleCostBuilder Builder(TTI); - SmallVector CommonMask(Mask.begin(), Mask.end()); Value *V1 = P1.dyn_cast(), *V2 = P2.dyn_cast(); - unsigned CommonVF = Mask.size(); - if (!V1 && !V2 && !P2.isNull()) { - // Shuffle 2 entry nodes. + unsigned CommonVF = 0; + if (!V1) { const TreeEntry *E = P1.get(); unsigned VF = E->getVectorFactor(); - const TreeEntry *E2 = P2.get(); - CommonVF = std::max(VF, E2->getVectorFactor()); - assert(all_of(Mask, - [=](int Idx) { - return Idx < 2 * static_cast(CommonVF); - }) && - "All elements in mask must be less than 2 * CommonVF."); - if (E->Scalars.size() == E2->Scalars.size()) { - SmallVector EMask = E->getCommonMask(); - SmallVector E2Mask = E2->getCommonMask(); - if (!EMask.empty() || !E2Mask.empty()) { - for (int &Idx : CommonMask) { - if (Idx == PoisonMaskElem) - continue; - if (Idx < static_cast(CommonVF) && !EMask.empty()) - Idx = EMask[Idx]; - else if (Idx >= static_cast(CommonVF)) - Idx = (E2Mask.empty() ? Idx - CommonVF : E2Mask[Idx - CommonVF]) + - E->Scalars.size(); - } + if (V2) { + unsigned V2VF = cast(V2->getType())->getNumElements(); + if (V2VF != VF && V2VF == E->Scalars.size()) + VF = E->Scalars.size(); + } else if (!P2.isNull()) { + const TreeEntry *E2 = P2.get(); + if (E->Scalars.size() == E2->Scalars.size()) + CommonVF = VF = E->Scalars.size(); + } else { + // P2 is empty, check that we have same node + reshuffle (if any). + if (E->Scalars.size() == Mask.size() && VF != Mask.size()) { + VF = E->Scalars.size(); + SmallVector CommonMask(Mask.begin(), Mask.end()); + ::addMask(CommonMask, E->getCommonMask()); + V1 = Constant::getNullValue( + FixedVectorType::get(E->Scalars.front()->getType(), VF)); + return BaseShuffleAnalysis::createShuffle( + V1, nullptr, CommonMask, Builder); } - CommonVF = E->Scalars.size(); } V1 = Constant::getNullValue( - FixedVectorType::get(E->Scalars.front()->getType(), CommonVF)); - V2 = getAllOnesValue( - *R.DL, FixedVectorType::get(E->Scalars.front()->getType(), CommonVF)); - } else if (!V1 && P2.isNull()) { - // Shuffle single entry node. - const TreeEntry *E = P1.get(); + FixedVectorType::get(E->Scalars.front()->getType(), VF)); + } + if (!V2 && !P2.isNull()) { + const TreeEntry *E = P2.get(); unsigned VF = E->getVectorFactor(); - CommonVF = VF; - assert( - all_of(Mask, - [=](int Idx) { return Idx < static_cast(CommonVF); }) && - "All elements in mask must be less than CommonVF."); - if (E->Scalars.size() == Mask.size() && VF != Mask.size()) { - SmallVector EMask = E->getCommonMask(); - assert(!EMask.empty() && "Expected non-empty common mask."); - for (int &Idx : CommonMask) { - if (Idx != PoisonMaskElem) - Idx = EMask[Idx]; - } + unsigned V1VF = cast(V1->getType())->getNumElements(); + if (!CommonVF && V1VF == E->Scalars.size()) CommonVF = E->Scalars.size(); - } - V1 = Constant::getNullValue( - FixedVectorType::get(E->Scalars.front()->getType(), CommonVF)); - } else if (V1 && P2.isNull()) { - // Shuffle single vector. - CommonVF = cast(V1->getType())->getNumElements(); - assert( - all_of(Mask, - [=](int Idx) { return Idx < static_cast(CommonVF); }) && - "All elements in mask must be less than CommonVF."); - } else if (V1 && !V2) { - // Shuffle vector and tree node. - unsigned VF = cast(V1->getType())->getNumElements(); - const TreeEntry *E2 = P2.get(); - CommonVF = std::max(VF, E2->getVectorFactor()); - assert(all_of(Mask, - [=](int Idx) { - return Idx < 2 * static_cast(CommonVF); - }) && - "All elements in mask must be less than 2 * CommonVF."); - if (E2->Scalars.size() == VF && VF != CommonVF) { - SmallVector E2Mask = E2->getCommonMask(); - assert(!E2Mask.empty() && "Expected non-empty common mask."); - for (int &Idx : CommonMask) { - if (Idx == PoisonMaskElem) - continue; - if (Idx >= static_cast(CommonVF)) - Idx = E2Mask[Idx - CommonVF] + VF; - } - CommonVF = VF; - } - V1 = Constant::getNullValue( - FixedVectorType::get(E2->Scalars.front()->getType(), CommonVF)); - V2 = getAllOnesValue( - *R.DL, - FixedVectorType::get(E2->Scalars.front()->getType(), CommonVF)); - } else { - assert(V1 && V2 && "Expected both vectors."); - unsigned VF = cast(V1->getType())->getNumElements(); - CommonVF = - std::max(VF, cast(V2->getType())->getNumElements()); - assert(all_of(Mask, - [=](int Idx) { - return Idx < 2 * static_cast(CommonVF); - }) && - "All elements in mask must be less than 2 * CommonVF."); - if (V1->getType() != V2->getType()) { - V1 = Constant::getNullValue(FixedVectorType::get( - cast(V1->getType())->getElementType(), CommonVF)); - V2 = getAllOnesValue( - *R.DL, FixedVectorType::get( - cast(V1->getType())->getElementType(), - CommonVF)); - } - } - InVectors.front() = Constant::getNullValue(FixedVectorType::get( - cast(V1->getType())->getElementType(), - CommonMask.size())); - if (InVectors.size() == 2) - InVectors.pop_back(); - return BaseShuffleAnalysis::createShuffle( - V1, V2, CommonMask, Builder); + if (CommonVF) + VF = CommonVF; + V2 = Constant::getNullValue( + FixedVectorType::get(E->Scalars.front()->getType(), VF)); + } + return BaseShuffleAnalysis::createShuffle(V1, V2, Mask, + Builder); } public: @@ -7512,7 +7415,7 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { } return ConstantVector::getSplat( ElementCount::getFixed(VL.size()), - getAllOnesValue(*R.DL, VL.front()->getType())); + Constant::getNullValue(VL.front()->getType())); } /// Finalize emission of the shuffles. InstructionCost @@ -7521,22 +7424,31 @@ class BoUpSLP::ShuffleCostEstimator : public BaseShuffleAnalysis { IsFinalized = true; if (Action) { const PointerUnion &Vec = InVectors.front(); - if (InVectors.size() == 2) + if (InVectors.size() == 2) { Cost += createShuffle(Vec, InVectors.back(), CommonMask); - else + InVectors.pop_back(); + } else { Cost += createShuffle(Vec, nullptr, CommonMask); + } for (unsigned Idx = 0, Sz = CommonMask.size(); Idx < Sz; ++Idx) if (CommonMask[Idx] != PoisonMaskElem) CommonMask[Idx] = Idx; assert(VF > 0 && "Expected vector length for the final value before action."); - Value *V = Vec.get(); + Value *V = Vec.dyn_cast(); + if (!Vec.isNull() && !V) + V = Constant::getNullValue(FixedVectorType::get( + Vec.get()->Scalars.front()->getType(), + CommonMask.size())); Action(V, CommonMask); - InVectors.front() = V; } ::addMask(CommonMask, ExtMask, /*ExtendingManyInputs=*/true); if (CommonMask.empty()) return Cost; + int Limit = CommonMask.size() * 2; + if (all_of(CommonMask, [=](int Idx) { return Idx < Limit; }) && + ShuffleVectorInst::isIdentityMask(CommonMask)) + return Cost; return Cost + createShuffle(InVectors.front(), InVectors.size() == 2 ? InVectors.back() : nullptr, @@ -7712,7 +7624,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, } if (NeedToShuffleReuses) ::addMask(Mask, E->ReuseShuffleIndices); - if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) + if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask)) CommonCost = TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FinalVecTy, Mask); assert((E->State == TreeEntry::Vectorize || @@ -8063,7 +7975,7 @@ BoUpSLP::getEntryCost(const TreeEntry *E, ArrayRef VectorizedVals, } else { SmallVector Mask; inversePermutation(OpTE->ReorderIndices, Mask); - if (ShuffleVectorInst::isReverseMask(Mask, Mask.size())) + if (ShuffleVectorInst::isReverseMask(Mask)) CCH = TTI::CastContextHint::Reversed; } } @@ -8922,7 +8834,9 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { unsigned VecVF = TE->getVectorFactor(); if (VF != VecVF && (any_of(Mask, [VF](int Idx) { return Idx >= static_cast(VF); }) || - !ShuffleVectorInst::isIdentityMask(Mask, VF))) { + (all_of(Mask, + [VF](int Idx) { return Idx < 2 * static_cast(VF); }) && + !ShuffleVectorInst::isIdentityMask(Mask)))) { SmallVector OrigMask(VecVF, PoisonMaskElem); std::copy(Mask.begin(), std::next(Mask.begin(), std::min(VF, VecVF)), OrigMask.begin()); @@ -8951,7 +8865,9 @@ InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { assert((TEs.size() == 1 || TEs.size() == 2) && "Expected exactly 1 or 2 tree entries."); if (TEs.size() == 1) { - if (!ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) { + int Limit = 2 * Mask.size(); + if (!all_of(Mask, [Limit](int Idx) { return Idx < Limit; }) || + !ShuffleVectorInst::isIdentityMask(Mask)) { InstructionCost C = TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FTy, Mask); LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C @@ -9727,7 +9643,7 @@ class BoUpSLP::ShuffleInstructionBuilder final : public BaseShuffleAnalysis { return V1; unsigned VF = Mask.size(); unsigned LocalVF = cast(V1->getType())->getNumElements(); - if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask, VF)) + if (VF == LocalVF && ShuffleVectorInst::isIdentityMask(Mask)) return V1; Value *Vec = Builder.CreateShuffleVector(V1, Mask); if (auto *I = dyn_cast(Vec)) { @@ -10140,7 +10056,9 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) { return false; unsigned I = *find_if_not(Mask, [](int Idx) { return Idx == PoisonMaskElem; }); - if (ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) + int Sz = Mask.size(); + if (all_of(Mask, [Sz](int Idx) { return Idx < 2 * Sz; }) && + ShuffleVectorInst::isIdentityMask(Mask)) std::iota(Mask.begin(), Mask.end(), 0); else std::fill(Mask.begin(), Mask.end(), I); @@ -10384,11 +10302,11 @@ ResTy BoUpSLP::processBuildVector(const TreeEntry *E, Args &...Params) { (ExtractShuffle.value_or(TTI::SK_PermuteTwoSrc) == TTI::SK_PermuteSingleSrc && none_of(ExtractMask, [&](int I) { return I >= EMSz; }) && - ShuffleVectorInst::isIdentityMask(ExtractMask, EMSz)) || + ShuffleVectorInst::isIdentityMask(ExtractMask)) || (GatherShuffle.value_or(TTI::SK_PermuteTwoSrc) == TTI::SK_PermuteSingleSrc && none_of(Mask, [&](int I) { return I >= MSz; }) && - ShuffleVectorInst::isIdentityMask(Mask, MSz)); + ShuffleVectorInst::isIdentityMask(Mask)); bool EnoughConstsForShuffle = IsSingleShuffle && (none_of(GatheredScalars, @@ -10681,7 +10599,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) { if ((!IsIdentity || Offset != 0 || !IsFirstUndef.all()) && NumElts != NumScalars) { if (IsFirstUndef.all()) { - if (!ShuffleVectorInst::isIdentityMask(InsertMask, NumElts)) { + if (!ShuffleVectorInst::isIdentityMask(InsertMask)) { SmallBitVector IsFirstPoison = isUndefVector(FirstInsert->getOperand(0), UseMask); if (!IsFirstPoison.all()) { @@ -11504,7 +11422,7 @@ Value *BoUpSLP::vectorizeTree( // non-resizing mask. if (Mask.size() != cast(Vals.front()->getType()) ->getNumElements() || - !ShuffleVectorInst::isIdentityMask(Mask, Mask.size())) + !ShuffleVectorInst::isIdentityMask(Mask)) return CreateShuffle(Vals.front(), nullptr, Mask); return Vals.front(); } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll index 99f03cdcf648c..d6196c84e8e2d 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll @@ -461,21 +461,44 @@ exit: define void @combine_load_factor2_i64(ptr noalias %p, ptr noalias %q) { ; CHECK-LABEL: @combine_load_factor2_i64( ; CHECK-NEXT: entry: +; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1 +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i64, ptr [[TMP2]], i32 0 +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <8 x i64>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> +; CHECK-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i64> [[WIDE_VEC]], <8 x i64> poison, <4 x i32> +; CHECK-NEXT: [[TMP4:%.*]] = add <4 x i64> [[STRIDED_VEC]], [[STRIDED_VEC1]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 +; CHECK-NEXT: store <4 x i64> [[TMP4]], ptr [[TMP6]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 +; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: middle.block: +; CHECK-NEXT: br i1 true, label [[EXIT:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 1024, [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[NEXTI:%.*]], [[LOOP]] ] +; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[NEXTI:%.*]], [[LOOP]] ] ; CHECK-NEXT: [[OFFSET0:%.*]] = shl i64 [[I]], 1 -; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[OFFSET0]] +; CHECK-NEXT: [[Q0:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET0]] ; CHECK-NEXT: [[X0:%.*]] = load i64, ptr [[Q0]], align 4 ; CHECK-NEXT: [[OFFSET1:%.*]] = add i64 [[OFFSET0]], 1 ; CHECK-NEXT: [[Q1:%.*]] = getelementptr i64, ptr [[P]], i64 [[OFFSET1]] ; CHECK-NEXT: [[X1:%.*]] = load i64, ptr [[Q1]], align 4 ; CHECK-NEXT: [[RES:%.*]] = add i64 [[X0]], [[X1]] -; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q:%.*]], i64 [[I]] +; CHECK-NEXT: [[DST:%.*]] = getelementptr i64, ptr [[Q]], i64 [[I]] ; CHECK-NEXT: store i64 [[RES]], ptr [[DST]], align 4 ; CHECK-NEXT: [[NEXTI]] = add i64 [[I]], 1 ; CHECK-NEXT: [[DONE:%.*]] = icmp eq i64 [[NEXTI]], 1024 -; CHECK-NEXT: br i1 [[DONE]], label [[EXIT:%.*]], label [[LOOP]] +; CHECK-NEXT: br i1 [[DONE]], label [[EXIT]], label [[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: exit: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll index eb3d395f4c6a6..c5981c44d96dd 100644 --- a/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll +++ b/llvm/test/Transforms/SLPVectorizer/X86/tiny-tree.ll @@ -241,10 +241,21 @@ define void @tiny_vector_gather(ptr %a, ptr %v1, ptr %v2) { ; CHECK-LABEL: @tiny_vector_gather( ; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[V1:%.*]], align 4 ; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr [[V2:%.*]], align 4 -; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> poison, i32 [[TMP1]], i32 0 -; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[TMP2]], i32 1 -; CHECK-NEXT: [[TMP5:%.*]] = shufflevector <8 x i32> [[TMP4]], <8 x i32> poison, <8 x i32> -; CHECK-NEXT: store <8 x i32> [[TMP5]], ptr [[A:%.*]], align 16 +; CHECK-NEXT: store i32 [[TMP1]], ptr [[A:%.*]], align 16 +; CHECK-NEXT: [[PTR1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 1 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR1]], align 4 +; CHECK-NEXT: [[PTR2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 2 +; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR2]], align 8 +; CHECK-NEXT: [[PTR3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 3 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR3]], align 4 +; CHECK-NEXT: [[PTR4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 4 +; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR4]], align 16 +; CHECK-NEXT: [[PTR5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 5 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR5]], align 4 +; CHECK-NEXT: [[PTR6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 6 +; CHECK-NEXT: store i32 [[TMP1]], ptr [[PTR6]], align 8 +; CHECK-NEXT: [[PTR7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 7 +; CHECK-NEXT: store i32 [[TMP2]], ptr [[PTR7]], align 4 ; CHECK-NEXT: ret void ; %1 = load i32, ptr %v1, align 4 diff --git a/llvm/unittests/IR/InstructionsTest.cpp b/llvm/unittests/IR/InstructionsTest.cpp index 20b8529b38632..637c69204addf 100644 --- a/llvm/unittests/IR/InstructionsTest.cpp +++ b/llvm/unittests/IR/InstructionsTest.cpp @@ -1024,141 +1024,71 @@ TEST(InstructionsTest, ShuffleMaskQueries) { Constant *C7 = ConstantInt::get(Int32Ty, 7); Constant *Identity = ConstantVector::get({C0, CU, C2, C3, C4}); - EXPECT_TRUE(ShuffleVectorInst::isIdentityMask( - Identity, cast(Identity->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask( - Identity, - cast(Identity->getType()) - ->getNumElements())); // identity is distinguished from select - EXPECT_FALSE(ShuffleVectorInst::isReverseMask( - Identity, cast(Identity->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( - Identity, cast(Identity->getType()) - ->getNumElements())); // identity is always single source - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( - Identity, cast(Identity->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( - Identity, cast(Identity->getType())->getNumElements())); + EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(Identity)); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Identity)); // identity is distinguished from select + EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Identity)); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Identity)); // identity is always single source + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Identity)); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Identity)); Constant *Select = ConstantVector::get({CU, C1, C5}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( - Select, cast(Select->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isSelectMask( - Select, cast(Select->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask( - Select, cast(Select->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask( - Select, cast(Select->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( - Select, cast(Select->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( - Select, cast(Select->getType())->getNumElements())); - + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Select)); + EXPECT_TRUE(ShuffleVectorInst::isSelectMask(Select)); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Select)); + EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Select)); + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Select)); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Select)); + Constant *Reverse = ConstantVector::get({C3, C2, C1, CU}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( - Reverse, cast(Reverse->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask( - Reverse, cast(Reverse->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isReverseMask( - Reverse, cast(Reverse->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( - Reverse, cast(Reverse->getType()) - ->getNumElements())); // reverse is always single source - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( - Reverse, cast(Reverse->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( - Reverse, cast(Reverse->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Reverse)); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Reverse)); + EXPECT_TRUE(ShuffleVectorInst::isReverseMask(Reverse)); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(Reverse)); // reverse is always single source + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Reverse)); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(Reverse)); Constant *SingleSource = ConstantVector::get({C2, C2, C0, CU}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( - SingleSource, - cast(SingleSource->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask( - SingleSource, - cast(SingleSource->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask( - SingleSource, - cast(SingleSource->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( - SingleSource, - cast(SingleSource->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( - SingleSource, - cast(SingleSource->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( - SingleSource, - cast(SingleSource->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(SingleSource)); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask(SingleSource)); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask(SingleSource)); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(SingleSource)); + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(SingleSource)); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(SingleSource)); Constant *ZeroEltSplat = ConstantVector::get({C0, C0, CU, C0}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( - ZeroEltSplat, - cast(ZeroEltSplat->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask( - ZeroEltSplat, - cast(ZeroEltSplat->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask( - ZeroEltSplat, - cast(ZeroEltSplat->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( - ZeroEltSplat, cast(ZeroEltSplat->getType()) - ->getNumElements())); // 0-splat is always single source - EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask( - ZeroEltSplat, - cast(ZeroEltSplat->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isTransposeMask( - ZeroEltSplat, - cast(ZeroEltSplat->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(ZeroEltSplat)); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask(ZeroEltSplat)); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask(ZeroEltSplat)); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ZeroEltSplat)); // 0-splat is always single source + EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ZeroEltSplat)); + EXPECT_FALSE(ShuffleVectorInst::isTransposeMask(ZeroEltSplat)); Constant *Transpose = ConstantVector::get({C0, C4, C2, C6}); - EXPECT_FALSE(ShuffleVectorInst::isIdentityMask( - Transpose, - cast(Transpose->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSelectMask( - Transpose, - cast(Transpose->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isReverseMask( - Transpose, - cast(Transpose->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask( - Transpose, - cast(Transpose->getType())->getNumElements())); - EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask( - Transpose, - cast(Transpose->getType())->getNumElements())); - EXPECT_TRUE(ShuffleVectorInst::isTransposeMask( - Transpose, - cast(Transpose->getType())->getNumElements())); + EXPECT_FALSE(ShuffleVectorInst::isIdentityMask(Transpose)); + EXPECT_FALSE(ShuffleVectorInst::isSelectMask(Transpose)); + EXPECT_FALSE(ShuffleVectorInst::isReverseMask(Transpose)); + EXPECT_FALSE(ShuffleVectorInst::isSingleSourceMask(Transpose)); + EXPECT_FALSE(ShuffleVectorInst::isZeroEltSplatMask(Transpose)); + EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(Transpose)); // More tests to make sure the logic is/stays correct... - EXPECT_TRUE(ShuffleVectorInst::isIdentityMask( - ConstantVector::get({CU, C1, CU, C3}), 4)); - EXPECT_TRUE(ShuffleVectorInst::isIdentityMask( - ConstantVector::get({C4, CU, C6, CU}), 4)); - - EXPECT_TRUE(ShuffleVectorInst::isSelectMask( - ConstantVector::get({C4, C1, C6, CU}), 4)); - EXPECT_TRUE(ShuffleVectorInst::isSelectMask( - ConstantVector::get({CU, C1, C6, C3}), 4)); - - EXPECT_TRUE(ShuffleVectorInst::isReverseMask( - ConstantVector::get({C7, C6, CU, C4}), 4)); - EXPECT_TRUE(ShuffleVectorInst::isReverseMask( - ConstantVector::get({C3, CU, C1, CU}), 4)); - - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( - ConstantVector::get({C7, C5, CU, C7}), 4)); - EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask( - ConstantVector::get({C3, C0, CU, C3}), 4)); - - EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask( - ConstantVector::get({C4, CU, CU, C4}), 4)); - EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask( - ConstantVector::get({CU, C0, CU, C0}), 4)); - - EXPECT_TRUE(ShuffleVectorInst::isTransposeMask( - ConstantVector::get({C1, C5, C3, C7}), 4)); - EXPECT_TRUE( - ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}), 2)); + EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({CU, C1, CU, C3}))); + EXPECT_TRUE(ShuffleVectorInst::isIdentityMask(ConstantVector::get({C4, CU, C6, CU}))); + + EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({C4, C1, C6, CU}))); + EXPECT_TRUE(ShuffleVectorInst::isSelectMask(ConstantVector::get({CU, C1, C6, C3}))); + + EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C7, C6, CU, C4}))); + EXPECT_TRUE(ShuffleVectorInst::isReverseMask(ConstantVector::get({C3, CU, C1, CU}))); + + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C7, C5, CU, C7}))); + EXPECT_TRUE(ShuffleVectorInst::isSingleSourceMask(ConstantVector::get({C3, C0, CU, C3}))); + + EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({C4, CU, CU, C4}))); + EXPECT_TRUE(ShuffleVectorInst::isZeroEltSplatMask(ConstantVector::get({CU, C0, CU, C0}))); + + EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C5, C3, C7}))); + EXPECT_TRUE(ShuffleVectorInst::isTransposeMask(ConstantVector::get({C1, C3}))); // Nothing special about the values here - just re-using inputs to reduce code. Constant *V0 = ConstantVector::get({C0, C1, C2, C3}); diff --git a/llvm/unittests/IR/ShuffleVectorInstTest.cpp b/llvm/unittests/IR/ShuffleVectorInstTest.cpp index 5d840ec8d6720..ba1b9807cffb4 100644 --- a/llvm/unittests/IR/ShuffleVectorInstTest.cpp +++ b/llvm/unittests/IR/ShuffleVectorInstTest.cpp @@ -14,84 +14,51 @@ using namespace llvm; namespace { TEST(ShuffleVectorInst, isIdentityMask) { - ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 4)); - ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 5)); - ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3})); + ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1})); + ASSERT_TRUE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3})); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 4)); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 3)); - - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3}, 5)); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 3, -1}, 6)); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, -1, 3}, 5)); - - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4}, 4)); - ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, 1, 2, 4})); + ASSERT_FALSE(ShuffleVectorInst::isIdentityMask({0, -1, 2, 4})); } TEST(ShuffleVectorInst, isSelectMask) { - ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 4)); - - ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3}, 5)); + ASSERT_TRUE(ShuffleVectorInst::isSelectMask({0, 5, 6, 3})); - ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isSelectMask({0, 1, 2, 3})); } TEST(ShuffleVectorInst, isReverseMask) { - ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 4)); - ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0})); + ASSERT_TRUE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0})); - ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isReverseMask({3, 2, 1, 0}, 5)); - ASSERT_FALSE(ShuffleVectorInst::isReverseMask({-1, -1, 1, 0}, 5)); - - ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isReverseMask({4, 3, 2, 1})); } TEST(ShuffleVectorInst, isZeroEltSplatMask) { - ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 4)); - ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 4)); - - ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0}, 5)); - ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1}, 5)); + ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, 0, 0, 0})); + ASSERT_TRUE(ShuffleVectorInst::isZeroEltSplatMask({0, -1, 0, -1})); - ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isZeroEltSplatMask({1, 1, 1, 1})); } TEST(ShuffleVectorInst, isTransposeMask) { - ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 4)); - ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 4)); + ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6})); + ASSERT_TRUE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7})); - ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 3)); - ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({0, 4, 2, 6}, 5)); - ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({1, 5, 3, 7}, 5)); - - ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8}, 4)); + ASSERT_FALSE(ShuffleVectorInst::isTransposeMask({2, 6, 4, 8})); } TEST(ShuffleVectorInst, isSpliceMask) { int Index; - ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 4, Index)); + ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, Index)); ASSERT_EQ(0, Index); - ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 7, Index)); + ASSERT_TRUE(ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, Index)); ASSERT_EQ(1, Index); - ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 3, Index)); - ASSERT_FALSE( - ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 6, Index)); - ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({0, 1, 2, 3}, 5, Index)); - ASSERT_FALSE( - ShuffleVectorInst::isSpliceMask({1, 2, 3, 4, 5, 6, 7}, 8, Index)); - - ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, 4, Index)); + ASSERT_FALSE(ShuffleVectorInst::isSpliceMask({4, 5, 6, 7}, Index)); } TEST(ShuffleVectorInst, isExtractSubvectorMask) {