diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index f18a3362d4af7..9a525d69b3ee8 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -307,6 +307,16 @@ class LLVM_ABI TargetLoweringBase { Expensive = 2 // Negated expression is more expensive. }; + /// Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is. + enum class ExtractSubvectorCost { + Free = 0, // Lowers to no instruction at all, e.g. a subregister copy. + Cheap = 1, // Lowers to at most one instruction, and may still be free if + // the target can fold the extract into the instruction + // consuming it (e.g. a widening op that reads the high half of + // a register). + Expensive = 2 // Needs a shuffle sequence that cannot be folded away. + }; + /// Enum of different potentially desirable ways to fold (and/or (setcc ...), /// (setcc ...)). enum AndOrSETCCFoldKind : uint8_t { @@ -3540,13 +3550,16 @@ class LLVM_ABI TargetLoweringBase { return false; } - /// Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type - /// from this source type with this index. This is needed because - /// EXTRACT_SUBVECTOR usually has custom lowering that depends on the index of - /// the first element, and only the target knows which lowering is cheap. - virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { - return false; + /// Return the cost of extracting a subvector of type \p ResVT from a vector + /// of type \p SrcVT, starting at element \p Index. + /// + /// Most callers only create a new EXTRACT_SUBVECTOR when the cost is at most + /// ExtractSubvectorCost::Cheap. This hook exists because EXTRACT_SUBVECTOR + /// usually has custom lowering that depends on the index of the first + /// element, so only the target knows which lowering is cheap. + virtual ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { + return ExtractSubvectorCost::Expensive; } /// Try to convert an extract element of a vector binary operation into an diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c669d8d70d103..a11e21769954f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -21922,7 +21922,8 @@ SDValue DAGCombiner::ForwardStoreValueToDirectLoad(LoadSDNode *LD) { InterVT.getVectorNumElements() - LDMemType.getVectorNumElements(); } - if (!TLI.isExtractSubvectorCheap(LDMemType, InterVT, ExtIdx)) + if (TLI.getExtractSubvectorCost(LDMemType, InterVT, ExtIdx) > + TargetLowering::ExtractSubvectorCost::Cheap) break; Val = DAG.getExtractSubvector(SDLoc(LD), LDMemType, DAG.getBitcast(InterVT, Val), ExtIdx); @@ -26341,7 +26342,8 @@ SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N, VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); VecIn2 = SDValue(); } else if (InVT1Size == VTSize * 2) { - if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems)) + if (TLI.getExtractSubvectorCost(VT, InVT1, NumElems) > + TargetLowering::ExtractSubvectorCost::Cheap) return SDValue(); if (!VecIn2.getNode()) { @@ -26380,7 +26382,8 @@ SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N, ConcatOps[0] = VecIn2; VecIn2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps); } else if (InVT1Size / VTSize > 1 && InVT1Size % VTSize == 0) { - if (!TLI.isExtractSubvectorCheap(VT, InVT1, NumElems) || + if (TLI.getExtractSubvectorCost(VT, InVT1, NumElems) > + TargetLowering::ExtractSubvectorCost::Cheap || !TLI.isTypeLegal(InVT1) || !TLI.isTypeLegal(InVT2)) return SDValue(); // If dest vector has less than two elements, then use shuffle and extract @@ -27856,7 +27859,8 @@ static SDValue narrowExtractedVectorBinOp(EVT VT, SDValue Src, unsigned Index, // bitcasted. unsigned ConcatOpNum = Index / VT.getVectorNumElements(); unsigned ExtBOIdx = ConcatOpNum * NarrowBVT.getVectorNumElements(); - if (TLI.isExtractSubvectorCheap(NarrowBVT, WideBVT, ExtBOIdx) && + if (TLI.getExtractSubvectorCost(NarrowBVT, WideBVT, ExtBOIdx) <= + TargetLowering::ExtractSubvectorCost::Cheap && BinOp.hasOneUse() && Src->hasOneUse()) { // extract (binop B0, B1), N --> binop (extract B0, N), (extract B1, N) SDValue NewExtIndex = DAG.getVectorIdxConstant(ExtBOIdx, DL); @@ -28067,7 +28071,8 @@ static SDValue foldExtractSubvectorFromShuffleVector(EVT NarrowVT, SDValue Src, // How many elements into the WideVT does this subvector start? int Index = NumEltsExtracted * OpSubvecIdx; // Bail out if the extraction isn't going to be cheap. - if (!TLI.isExtractSubvectorCheap(NarrowVT, WideVT, Index)) + if (TLI.getExtractSubvectorCost(NarrowVT, WideVT, Index) > + TargetLowering::ExtractSubvectorCost::Cheap) return SDValue(); } @@ -28190,8 +28195,9 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) { uint64_t NewExtIdx = InnerExtIdx + ExtIdx; if (V.getValueType().isScalableVector() == NVT.isScalableVector() && NewExtIdx % NVT.getVectorMinNumElements() == 0 && - TLI.isExtractSubvectorCheap(NVT, V.getOperand(0).getValueType(), - NewExtIdx) && + TLI.getExtractSubvectorCost(NVT, V.getOperand(0).getValueType(), + NewExtIdx) <= + TargetLowering::ExtractSubvectorCost::Cheap && TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) return DAG.getExtractSubvector(DL, NVT, V.getOperand(0), NewExtIdx); } @@ -28200,7 +28206,8 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) { if (V.getOpcode() == ISD::SPLAT_VECTOR) if ((DAG.isConstantValueOfAnyType(V.getOperand(0)) && !(NVT.isScalableVector() && - TLI.isExtractSubvectorCheap(NVT, V.getValueType(), ExtIdx))) || + TLI.getExtractSubvectorCost(NVT, V.getValueType(), ExtIdx) <= + TargetLowering::ExtractSubvectorCost::Cheap)) || V.hasOneUse()) if (!LegalOperations || TLI.isOperationLegal(ISD::SPLAT_VECTOR, NVT)) return DAG.getSplatVector(NVT, DL, V.getOperand(0)); @@ -28224,7 +28231,8 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) { unsigned InsIdx = V.getConstantOperandVal(2); unsigned NumSubElts = NVT.getVectorMinNumElements(); if (InsIdx <= ExtIdx && (ExtIdx + NumSubElts) <= (InsIdx + NumInsElts) && - TLI.isExtractSubvectorCheap(NVT, InsSubVT, ExtIdx - InsIdx) && + TLI.getExtractSubvectorCost(NVT, InsSubVT, ExtIdx - InsIdx) <= + TargetLowering::ExtractSubvectorCost::Cheap && InsSubVT.isFixedLengthVector() && NVT.isFixedLengthVector() && V.getValueType().isFixedLengthVector()) return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NVT, InsSub, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 4f5271f4c1d91..b9d0ed344663b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -14240,7 +14240,8 @@ SelectionDAG::matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, EVT OpVT = Op.getValueType(); EVT OpSVT = OpVT.getScalarType(); EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts); - if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0)) + if (TLI->getExtractSubvectorCost(SubVT, OpVT, 0) > + TargetLowering::ExtractSubvectorCost::Cheap) return SDValue(); BinOp = (ISD::NodeType)CandidateBinOp; return getExtractSubvector(SDLoc(Op), SubVT, Op, 0); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 8bcd90c2a1294..da29cd751c80e 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -20311,12 +20311,18 @@ bool AArch64TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, return Shift < 3; } -bool AArch64TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +AArch64TargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) - return false; + return ExtractSubvectorCost::Expensive; + + if (Index == 0) + return ExtractSubvectorCost::Free; - return (Index == 0 || Index == ResVT.getVectorMinNumElements()); + if (Index == ResVT.getVectorMinNumElements()) + return ExtractSubvectorCost::Cheap; + return ExtractSubvectorCost::Expensive; } bool AArch64TargetLowering::shouldOptimizeMulOverflowWithZeroHighBits( diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 35f7ef0e2151e..66a6261d2a991 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -329,10 +329,10 @@ class AArch64TargetLowering : public TargetLowering { bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override; - /// Return true if EXTRACT_SUBVECTOR is cheap for this result type - /// with this index. - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + /// Return the cost of EXTRACT_SUBVECTOR for this result type with this + /// index. + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override { diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index f53fd0d74e48e..7fd90e73297cd 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -2410,13 +2410,16 @@ bool SITargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, return true; } -bool SITargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +SITargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) - return false; + return ExtractSubvectorCost::Expensive; // TODO: Add more cases that are cheap. - return Index == 0; + if (Index == 0) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; } bool SITargetLowering::isExtractVecEltCheap(EVT VT, unsigned Index) const { diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 3e0e5da94471f..6de650561ef5a 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -405,8 +405,8 @@ class SITargetLowering final : public AMDGPUTargetLowering { bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override; - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; bool isExtractVecEltCheap(EVT VT, unsigned Index) const override; bool isTypeDesirableForOp(unsigned Op, EVT VT) const override; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index e2b39726c7c0d..3a31e072831aa 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -21496,12 +21496,15 @@ bool ARMTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm, return true; } -bool ARMTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +ARMTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) - return false; + return ExtractSubvectorCost::Expensive; - return (Index == 0 || Index == ResVT.getVectorNumElements()); + if (Index == 0 || Index == ResVT.getVectorNumElements()) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; } Instruction *ARMTargetLowering::makeDMB(IRBuilderBase &Builder, diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index 10f5442d7429b..2bbb91a8758e2 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -328,10 +328,10 @@ class VectorType; bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override; - /// Return true if EXTRACT_SUBVECTOR is cheap for this result type - /// with this index. - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + /// Return the cost of EXTRACT_SUBVECTOR for this result type with this + /// index. + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override { diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index ac3ffa4b9bb0f..44b304919b181 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -2149,18 +2149,21 @@ bool HexagonTargetLowering::shouldExpandBuildVectorWithShuffles(EVT VT, return false; } -bool HexagonTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +HexagonTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { assert(ResVT.getVectorElementType() == SrcVT.getVectorElementType()); if (!ResVT.isSimple() || !SrcVT.isSimple()) - return false; + return ExtractSubvectorCost::Expensive; MVT ResTy = ResVT.getSimpleVT(), SrcTy = SrcVT.getSimpleVT(); if (ResTy.getVectorElementType() != MVT::i1) - return true; + return ExtractSubvectorCost::Free; // Non-HVX bool vectors are relatively cheap. - return SrcTy.getVectorNumElements() <= 8; + if (SrcTy.getVectorNumElements() <= 8) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; } bool HexagonTargetLowering::isTargetCanonicalConstantNode(SDValue Op) const { diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h index cf2263fdc2ad8..8e7d31b20a892 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -78,8 +78,8 @@ class HexagonTargetLowering : public TargetLowering { // Should we expand the build vector with shuffles? bool shouldExpandBuildVectorWithShuffles(EVT VT, unsigned DefinedValues) const override; - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; bool isTargetCanonicalConstantNode(SDValue Op) const override; diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 38f46a9fc9855..e4a369c096545 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -11787,13 +11787,16 @@ bool LoongArchTargetLowering::shouldScalarizeBinop(SDValue VecOp) const { return isOperationLegalOrCustomOrPromote(Opc, ScalarVT); } -bool LoongArchTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +LoongArchTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) - return false; + return ExtractSubvectorCost::Expensive; // Extract a 128-bit subvector from index 0 of a 256-bit vector is free. - return Index == 0; + if (Index == 0) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; } bool LoongArchTargetLowering::isExtractVecEltCheap(EVT VT, diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index 32e32ab9148af..b30bdeef1812c 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -169,8 +169,8 @@ class LoongArchTargetLowering : public TargetLowering { unsigned Depth) const override; bool shouldScalarizeBinop(SDValue VecOp) const override; - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; bool isExtractVecEltCheap(EVT VT, unsigned Index) const override; /// Check if a constant splat can be generated using [x]vldi, where imm[12] diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 1c5a8e4b79483..ea4803e59ebc1 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -2871,32 +2871,36 @@ bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, } // TODO: This is very conservative. -bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +RISCVTargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && - (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) - return (Index % ResVT.getVectorNumElements()) == 0; + (ResVT == MVT::v4i8 || ResVT == MVT::v2i16)) { + if ((Index % ResVT.getVectorNumElements()) == 0) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; + } if (!Subtarget.hasVInstructions()) - return false; + return ExtractSubvectorCost::Expensive; if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) - return false; + return ExtractSubvectorCost::Expensive; // Extracts from index 0 are just subreg extracts. if (Index == 0) - return true; + return ExtractSubvectorCost::Free; // Only support extracting a fixed from a fixed vector for now. if (ResVT.isScalableVector() || SrcVT.isScalableVector()) - return false; + return ExtractSubvectorCost::Expensive; EVT EltVT = ResVT.getVectorElementType(); assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node"); // The smallest type we can slide is i8. if (EltVT == MVT::i1) - return false; + return ExtractSubvectorCost::Expensive; unsigned ResElts = ResVT.getVectorNumElements(); unsigned SrcElts = SrcVT.getVectorNumElements(); @@ -2909,7 +2913,7 @@ bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, // Index ensures we can use a vslidedown.vi. // TODO: We can generalize this when the exact VLEN is known. if (Index + ResElts <= MinVLMAX && Index < 31) - return true; + return ExtractSubvectorCost::Free; // Convervatively only handle extracting half of a vector. // TODO: We can do arbitrary slidedowns, but for now only support extracting @@ -2917,7 +2921,9 @@ bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, // TODO: For sizes which aren't multiples of VLEN sizes, this may not be // a cheap extract. However, this case is important in practice for // shuffled extracts of longer vectors. How resolve? - return (ResElts * 2) == SrcElts && Index == ResElts; + if ((ResElts * 2) == SrcElts && Index == ResElts) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; } MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 2ca7c392639f7..972cc256a3386 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -64,8 +64,8 @@ class RISCVTargetLowering : public TargetLowering { int getLegalZfaFPImm(const APFloat &Imm, EVT VT) const; bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override; - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; bool isIntDivCheap(EVT VT, AttributeList Attr) const override; diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 0afc81ccad9e0..f62bf664d970a 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3611,19 +3611,26 @@ bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT, (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2(); } -bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const { +TargetLowering::ExtractSubvectorCost +X86TargetLowering::getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const { if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT)) - return false; + return ExtractSubvectorCost::Expensive; // Mask vectors support all subregister combinations and operations that // extract half of vector. - if (ResVT.getVectorElementType() == MVT::i1) - return Index == 0 || - ((ResVT.getSizeInBits() * 2 == SrcVT.getSizeInBits()) && - (Index == ResVT.getVectorNumElements())); - - return (Index % ResVT.getVectorNumElements()) == 0; + if (ResVT.getVectorElementType() == MVT::i1) { + if (Index == 0 || ((ResVT.getSizeInBits() * 2 == SrcVT.getSizeInBits()) && + (Index == ResVT.getVectorNumElements()))) + return ExtractSubvectorCost::Free; + return ExtractSubvectorCost::Expensive; + } + + if (Index == 0) + return ExtractSubvectorCost::Free; + else if ((Index % ResVT.getVectorNumElements()) == 0) + return ExtractSubvectorCost::Cheap; + return ExtractSubvectorCost::Expensive; } bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const { diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 798050028c15a..a4b4e6f32591b 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -576,10 +576,10 @@ namespace llvm { bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override; - /// Return true if EXTRACT_SUBVECTOR is cheap for this result type - /// with this index. - bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, - unsigned Index) const override; + /// Return the cost of EXTRACT_SUBVECTOR for this result type with this + /// index. + ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, + unsigned Index) const override; /// Scalar ops always have equal or better analysis/performance/power than /// the vector equivalent, so this always makes sense if the scalar op is