diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index 624302bc6d0a3..fad08261e9981 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -845,34 +845,6 @@ class TargetTransformInfoImplBase { return 1; } - virtual InstructionCost - getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const { - return 1; - } - - virtual InstructionCost - getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, - bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const { - return 1; - } - - virtual InstructionCost - getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const { - return 1; - } - - virtual InstructionCost - getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, - bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const { - return InstructionCost::getInvalid(); - } - virtual InstructionCost getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, @@ -932,6 +904,11 @@ class TargetTransformInfoImplBase { virtual InstructionCost getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { + unsigned IID = MICA.getID(); + bool IsStrided = IID == Intrinsic::experimental_vp_strided_load || + IID == Intrinsic::experimental_vp_strided_store; + if (IsStrided) + return InstructionCost::getInvalid(); return 1; } virtual InstructionCost getCallInstrCost(Function *F, Type *RetTy, diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 4d9cfea5d1bab..44ea4e3c6cbea 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1557,55 +1557,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { return Cost; } - InstructionCost - getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override { - Type *DataTy = MICA.getDataType(); - Align Alignment = MICA.getAlignment(); - unsigned Opcode = MICA.getID() == Intrinsic::masked_load - ? Instruction::Load - : Instruction::Store; - // TODO: Pass on AddressSpace when we have test coverage. - return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false, - CostKind); - } - - InstructionCost - getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, - bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const override { - return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask, - true, CostKind); - } - - InstructionCost - getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override { - unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload - ? Instruction::Load - : Instruction::Store; - Type *DataTy = MICA.getDataType(); - bool VariableMask = MICA.getVariableMask(); - Align Alignment = MICA.getAlignment(); - // Treat expand load/compress store as gather/scatter operation. - // TODO: implement more precise cost estimation for these intrinsics. - return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask, - /*IsGatherScatter*/ true, CostKind); - } - - InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, - const Value *Ptr, bool VariableMask, - Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I) const override { - // For a target without strided memory operations (or for an illegal - // operation type on one which does), assume we lower to a gather/scatter - // operation. (Which may in turn be scalarized.) - return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); - } - InstructionCost getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, @@ -3054,8 +3005,6 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { TTI::TargetCostKind CostKind) const override { unsigned Id = MICA.getID(); Type *DataTy = MICA.getDataType(); - const Value *Ptr = MICA.getPointer(); - const Instruction *I = MICA.getInst(); bool VariableMask = MICA.getVariableMask(); Align Alignment = MICA.getAlignment(); @@ -3065,26 +3014,47 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { unsigned Opcode = Id == Intrinsic::experimental_vp_strided_load ? Instruction::Load : Instruction::Store; - return thisT()->getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + // For a target without strided memory operations (or for an illegal + // operation type on one which does), assume we lower to a gather/scatter + // operation. (Which may in turn be scalarized.) + return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, + VariableMask, true, CostKind); } - case Intrinsic::masked_scatter: - case Intrinsic::masked_gather: case Intrinsic::vp_scatter: - case Intrinsic::vp_gather: { - unsigned Opcode = - (Id == Intrinsic::masked_gather || Id == Intrinsic::vp_gather) - ? Instruction::Load - : Instruction::Store; - return thisT()->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + case Intrinsic::vp_gather: + case Intrinsic::masked_scatter: + case Intrinsic::masked_gather: { + unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather || + MICA.getID() == Intrinsic::vp_gather) + ? Instruction::Load + : Instruction::Store; + + return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, + VariableMask, true, CostKind); } + + case Intrinsic::vp_load: + case Intrinsic::vp_store: + return InstructionCost::getInvalid(); case Intrinsic::masked_load: - case Intrinsic::masked_store: - return thisT()->getMaskedMemoryOpCost(MICA, CostKind); + case Intrinsic::masked_store: { + unsigned Opcode = + Id == Intrinsic::masked_load ? Instruction::Load : Instruction::Store; + // TODO: Pass on AddressSpace when we have test coverage. + return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, true, false, + CostKind); + } case Intrinsic::masked_compressstore: - case Intrinsic::masked_expandload: - return thisT()->getExpandCompressMemoryOpCost(MICA, CostKind); + case Intrinsic::masked_expandload: { + unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload + ? Instruction::Load + : Instruction::Store; + // Treat expand load/compress store as gather/scatter operation. + // TODO: implement more precise cost estimation for these intrinsics. + return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, + VariableMask, + /*IsGatherScatter*/ true, CostKind); + } case Intrinsic::vp_load_ff: return InstructionCost::getInvalid(); default: diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 3a5f1499f9d2d..e8bbce202b407 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -4746,13 +4746,27 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const { return ST->hasSVE(); } +InstructionCost +AArch64TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + switch (MICA.getID()) { + case Intrinsic::masked_scatter: + case Intrinsic::masked_gather: + return getGatherScatterOpCost(MICA, CostKind); + case Intrinsic::masked_load: + case Intrinsic::masked_store: + return getMaskedMemoryOpCost(MICA, CostKind); + } + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); +} + InstructionCost AArch64TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { Type *Src = MICA.getDataType(); if (useNeonVector(Src)) - return BaseT::getMaskedMemoryOpCost(MICA, CostKind); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); auto LT = getTypeLegalizationCost(Src); if (!LT.first.isValid()) return InstructionCost::getInvalid(); @@ -4794,12 +4808,21 @@ static unsigned getSVEGatherScatterOverhead(unsigned Opcode, } } -InstructionCost AArch64TTIImpl::getGatherScatterOpCost( - unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { +InstructionCost +AArch64TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + + unsigned Opcode = (MICA.getID() == Intrinsic::masked_gather || + MICA.getID() == Intrinsic::vp_gather) + ? Instruction::Load + : Instruction::Store; + + Type *DataTy = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + const Instruction *I = MICA.getInst(); + if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy)) - return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); auto *VT = cast(DataTy); auto LT = getTypeLegalizationCost(DataTy); if (!LT.first.isValid()) diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index fe3bb5e7981d2..7ca8482e152d1 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -188,14 +188,14 @@ class AArch64TTIImpl final : public BasicTTIImplBase { unsigned Opcode2) const; InstructionCost - getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override; + getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const override; - InstructionCost - getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, - bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const override; + InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; + + InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; bool isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst, Type *Src) const; diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp index fdb0ec40cb41f..a48589cd52bde 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp @@ -1631,6 +1631,20 @@ InstructionCost ARMTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, CostKind, OpInfo, I); } +InstructionCost +ARMTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + switch (MICA.getID()) { + case Intrinsic::masked_scatter: + case Intrinsic::masked_gather: + return getGatherScatterOpCost(MICA, CostKind); + case Intrinsic::masked_load: + case Intrinsic::masked_store: + return getMaskedMemoryOpCost(MICA, CostKind); + } + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); +} + InstructionCost ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { @@ -1647,7 +1661,7 @@ ARMTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, return ST->getMVEVectorCostFactor(CostKind); } if (!isa(Src)) - return BaseT::getMaskedMemoryOpCost(MICA, CostKind); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); // Scalar cost, which is currently very high due to the efficiency of the // generated code. return cast(Src)->getNumElements() * 8; @@ -1694,13 +1708,19 @@ InstructionCost ARMTTIImpl::getInterleavedMemoryOpCost( UseMaskForCond, UseMaskForGaps); } -InstructionCost ARMTTIImpl::getGatherScatterOpCost( - unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { +InstructionCost +ARMTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + + Type *DataTy = MICA.getDataType(); + const Value *Ptr = MICA.getPointer(); + bool VariableMask = MICA.getVariableMask(); + Align Alignment = MICA.getAlignment(); + const Instruction *I = MICA.getInst(); + using namespace PatternMatch; if (!ST->hasMVEIntegerOps() || !EnableMaskedGatherScatters) - return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); assert(DataTy->isVectorTy() && "Can't do gather/scatters on scalar!"); auto *VTy = cast(DataTy); diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 30f2151b41239..a23256364dd9a 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -279,19 +279,19 @@ class ARMTTIImpl final : public BasicTTIImplBase { const Instruction *I = nullptr) const override; InstructionCost - getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override; + getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const override; + + InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond = false, bool UseMaskForGaps = false) const override; - InstructionCost - getGatherScatterOpCost(unsigned Opcode, Type *DataTy, const Value *Ptr, - bool VariableMask, Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const override; + InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy, diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp index 3f84cbb6555ed..75262226d1e8f 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp @@ -223,10 +223,24 @@ InstructionCost HexagonTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, OpInfo, I); } +InstructionCost +HexagonTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + switch (MICA.getID()) { + case Intrinsic::masked_scatter: + case Intrinsic::masked_gather: + return getGatherScatterOpCost(MICA, CostKind); + case Intrinsic::masked_load: + case Intrinsic::masked_store: + return getMaskedMemoryOpCost(MICA, CostKind); + } + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); +} + InstructionCost HexagonTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { - return BaseT::getMaskedMemoryOpCost(MICA, CostKind); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); } InstructionCost @@ -238,11 +252,10 @@ HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, return 1; } -InstructionCost HexagonTTIImpl::getGatherScatterOpCost( - unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { - return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); +InstructionCost +HexagonTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); } InstructionCost HexagonTTIImpl::getInterleavedMemoryOpCost( diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h index 67388984bb3e3..88bad14c63b86 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -120,18 +120,17 @@ class HexagonTTIImpl final : public BasicTTIImplBase { TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I = nullptr) const override; InstructionCost - getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override; + getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const override; + InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, VectorType *SrcTy, ArrayRef Mask, TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, ArrayRef Args = {}, const Instruction *CxtI = nullptr) const override; - InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, - const Value *Ptr, bool VariableMask, - Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I) const override; + InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp index e697b9810e824..c0b94a2181c99 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp @@ -1022,6 +1022,22 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, return getMemoryOpCost(Instruction::Load, DataTy, Alignment, AS, CostKind, {TTI::OK_AnyValue, TTI::OP_None}, nullptr); } + case Intrinsic::experimental_vp_strided_load: + case Intrinsic::experimental_vp_strided_store: + return getStridedMemoryOpCost(MICA, CostKind); + case Intrinsic::masked_compressstore: + case Intrinsic::masked_expandload: + return getExpandCompressMemoryOpCost(MICA, CostKind); + case Intrinsic::vp_scatter: + case Intrinsic::vp_gather: + case Intrinsic::masked_scatter: + case Intrinsic::masked_gather: + return getGatherScatterOpCost(MICA, CostKind); + case Intrinsic::vp_load: + case Intrinsic::vp_store: + case Intrinsic::masked_load: + case Intrinsic::masked_store: + return getMaskedMemoryOpCost(MICA, CostKind); } return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); } @@ -1029,15 +1045,16 @@ RISCVTTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, InstructionCost RISCVTTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { - unsigned Opcode = MICA.getID() == Intrinsic::masked_load ? Instruction::Load - : Instruction::Store; + unsigned IID = MICA.getID(); + bool IsLoad = IID == Intrinsic::masked_load || IID == Intrinsic::vp_load; + unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store; Type *Src = MICA.getDataType(); Align Alignment = MICA.getAlignment(); unsigned AddressSpace = MICA.getAddressSpace(); if (!isLegalMaskedLoadStore(Src, Alignment) || CostKind != TTI::TCK_RecipThroughput) - return BaseT::getMaskedMemoryOpCost(MICA, CostKind); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); return getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); } @@ -1139,19 +1156,24 @@ InstructionCost RISCVTTIImpl::getInterleavedMemoryOpCost( return MemCost + ShuffleCost; } -InstructionCost RISCVTTIImpl::getGatherScatterOpCost( - unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { +InstructionCost +RISCVTTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + + bool IsLoad = MICA.getID() == Intrinsic::masked_gather || + MICA.getID() == Intrinsic::vp_gather; + unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store; + Type *DataTy = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + const Instruction *I = MICA.getInst(); if (CostKind != TTI::TCK_RecipThroughput) - return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); if ((Opcode == Instruction::Load && !isLegalMaskedGather(DataTy, Align(Alignment))) || (Opcode == Instruction::Store && !isLegalMaskedScatter(DataTy, Align(Alignment)))) - return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); // Cost is proportional to the number of memory operations implied. For // scalable vectors, we use an estimate on that number since we don't @@ -1178,7 +1200,7 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost( (Opcode == Instruction::Load && isLegalMaskedExpandLoad(DataTy, Alignment)); if (!IsLegal || CostKind != TTI::TCK_RecipThroughput) - return BaseT::getExpandCompressMemoryOpCost(MICA, CostKind); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); // Example compressstore sequence: // vsetivli zero, 8, e32, m2, ta, ma (ignored) // vcompress.vm v10, v8, v0 @@ -1207,14 +1229,22 @@ InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost( LT.first * getRISCVInstructionCost(Opcodes, LT.second, CostKind); } -InstructionCost RISCVTTIImpl::getStridedMemoryOpCost( - unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { +InstructionCost +RISCVTTIImpl::getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + + unsigned Opcode = MICA.getID() == Intrinsic::experimental_vp_strided_load + ? Instruction::Load + : Instruction::Store; + + Type *DataTy = MICA.getDataType(); + Align Alignment = MICA.getAlignment(); + const Instruction *I = MICA.getInst(); + if (((Opcode == Instruction::Load || Opcode == Instruction::Store) && !isLegalStridedLoadStore(DataTy, Alignment)) || (Opcode != Instruction::Load && Opcode != Instruction::Store)) - return BaseT::getStridedMemoryOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment, CostKind, I); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); if (CostKind == TTI::TCK_CodeSize) return TTI::TCC_Basic; diff --git a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h index 2a73fe6255382..370961f24ab7c 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h +++ b/llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h @@ -149,7 +149,7 @@ class RISCVTTIImpl final : public BasicTTIImplBase { InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override; + TTI::TargetCostKind CostKind) const; InstructionCost getPointersChainCost(ArrayRef Ptrs, const Value *Base, @@ -194,21 +194,15 @@ class RISCVTTIImpl final : public BasicTTIImplBase { Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, bool UseMaskForCond = false, bool UseMaskForGaps = false) const override; - InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, - const Value *Ptr, bool VariableMask, - Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I) const override; + InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override; + TTI::TargetCostKind CostKind) const; - InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy, - const Value *Ptr, bool VariableMask, - Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I) const override; + InstructionCost getStridedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getCostOfKeepingLiveOverCall(ArrayRef Tys) const override; diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index a9dc96b53d530..9fb97918cb71a 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -5410,6 +5410,20 @@ InstructionCost X86TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, return Cost; } +InstructionCost +X86TTIImpl::getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + switch (MICA.getID()) { + case Intrinsic::masked_scatter: + case Intrinsic::masked_gather: + return getGatherScatterOpCost(MICA, CostKind); + case Intrinsic::masked_load: + case Intrinsic::masked_store: + return getMaskedMemoryOpCost(MICA, CostKind); + } + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); +} + InstructionCost X86TTIImpl::getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, TTI::TargetCostKind CostKind) const { @@ -6258,10 +6272,15 @@ InstructionCost X86TTIImpl::getGSVectorCost(unsigned Opcode, } /// Calculate the cost of Gather / Scatter operation -InstructionCost X86TTIImpl::getGatherScatterOpCost( - unsigned Opcode, Type *SrcVTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, - const Instruction *I = nullptr) const { +InstructionCost +X86TTIImpl::getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const { + bool IsLoad = MICA.getID() == Intrinsic::masked_gather || + MICA.getID() == Intrinsic::vp_gather; + unsigned Opcode = IsLoad ? Instruction::Load : Instruction::Store; + Type *SrcVTy = MICA.getDataType(); + const Value *Ptr = MICA.getPointer(); + Align Alignment = MICA.getAlignment(); if ((Opcode == Instruction::Load && (!isLegalMaskedGather(SrcVTy, Align(Alignment)) || forceScalarizeMaskedGather(cast(SrcVTy), @@ -6270,8 +6289,7 @@ InstructionCost X86TTIImpl::getGatherScatterOpCost( (!isLegalMaskedScatter(SrcVTy, Align(Alignment)) || forceScalarizeMaskedScatter(cast(SrcVTy), Align(Alignment))))) - return BaseT::getGatherScatterOpCost(Opcode, SrcVTy, Ptr, VariableMask, - Alignment, CostKind, I); + return BaseT::getMemIntrinsicInstrCost(MICA, CostKind); assert(SrcVTy->isVectorTy() && "Unexpected data type for Gather/Scatter"); PointerType *PtrTy = dyn_cast(Ptr->getType()); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index d6dea9427990b..9b68909df9363 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -182,14 +182,14 @@ class X86TTIImpl final : public BasicTTIImplBase { TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None}, const Instruction *I = nullptr) const override; + InstructionCost - getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, - TTI::TargetCostKind CostKind) const override; - InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy, - const Value *Ptr, bool VariableMask, - Align Alignment, - TTI::TargetCostKind CostKind, - const Instruction *I) const override; + getMemIntrinsicInstrCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const override; + InstructionCost getMaskedMemoryOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; + InstructionCost getGatherScatterOpCost(const MemIntrinsicCostAttributes &MICA, + TTI::TargetCostKind CostKind) const; InstructionCost getPointersChainCost(ArrayRef Ptrs, const Value *Base, const TTI::PointersChainInfo &Info, Type *AccessTy,