diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index f613cc3fae94db..13c9a2eaeefafd 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -1166,9 +1166,21 @@ class TargetTransformInfo { /// \return The expected cost of vector Insert and Extract. /// Use -1 to indicate that there is no information on the index value. + /// This is used when the instruction is not available; a typical use + /// case is to provision the cost of vectorization/scalarization in + /// vectorizer passes. InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index = -1) const; + /// \return The expected cost of vector Insert and Extract. + /// This is used when instruction is available, and implementation + /// asserts 'I' is not nullptr. + /// + /// A typical suitable use case is cost estimation when vector instruction + /// exists (e.g., from basic blocks during transformation). + InstructionCost getVectorInstrCost(const Instruction *I, Type *Val, + unsigned Index = -1) const; + /// \return The cost of replication shuffle of \p VF elements typed \p EltTy /// \p ReplicationFactor times. /// @@ -1747,6 +1759,8 @@ class TargetTransformInfo::Concept { const Instruction *I) = 0; virtual InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) = 0; + virtual InstructionCost getVectorInstrCost(const Instruction *I, Type *Val, + unsigned Index) = 0; virtual InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, @@ -2305,6 +2319,10 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept { unsigned Index) override { return Impl.getVectorInstrCost(Opcode, Val, Index); } + InstructionCost getVectorInstrCost(const Instruction *I, Type *Val, + unsigned Index) override { + return Impl.getVectorInstrCost(I, Val, Index); + } InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h index f3376a3982a6c5..514a970c599974 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -575,6 +575,11 @@ class TargetTransformInfoImplBase { return 1; } + InstructionCost getVectorInstrCost(const Instruction *I, Type *Val, + unsigned Index) const { + return 1; + } + unsigned getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, TTI::TargetCostKind CostKind) { @@ -1148,7 +1153,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase { if (auto *CI = dyn_cast(IE->getOperand(2))) if (CI->getValue().getActiveBits() <= 32) Idx = CI->getZExtValue(); - return TargetTTI->getVectorInstrCost(Opcode, Ty, Idx); + return TargetTTI->getVectorInstrCost(IE, Ty, Idx); } case Instruction::ShuffleVector: { auto *Shuffle = dyn_cast(U); @@ -1238,7 +1243,7 @@ class TargetTransformInfoImplCRTPBase : public TargetTransformInfoImplBase { if (CI->getValue().getActiveBits() <= 32) Idx = CI->getZExtValue(); Type *DstTy = U->getOperand(0)->getType(); - return TargetTTI->getVectorInstrCost(Opcode, DstTy, Idx); + return TargetTTI->getVectorInstrCost(EEI, DstTy, Idx); } } // By default, just classify everything as 'basic'. diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h index 33c93e4c56acc7..2b092d48121e30 100644 --- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h +++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h @@ -1159,6 +1159,11 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase { return LT.first; } + InstructionCost getVectorInstrCost(const Instruction *I, Type *Val, + unsigned Index) { + return thisT()->getVectorInstrCost(I->getOpcode(), Val, Index); + } + InstructionCost getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index 4d3d549803db10..c86279eae9a524 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -869,11 +869,26 @@ InstructionCost TargetTransformInfo::getCmpSelInstrCost( InstructionCost TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) const { + // FIXME: Assert that Opcode is either InsertElement or ExtractElement. + // This is mentioned in the interface description and respected by all + // callers, but never asserted upon. InstructionCost Cost = TTIImpl->getVectorInstrCost(Opcode, Val, Index); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } +InstructionCost TargetTransformInfo::getVectorInstrCost(const Instruction *I, + Type *Val, + unsigned Index) const { + assert((I != nullptr) && "Expect not-null instruction pointer"); + // FIXME: Assert that Opcode is either InsertElement or ExtractElement. + // This is mentioned in the interface description and respected by all + // callers, but never asserted upon. + InstructionCost Cost = TTIImpl->getVectorInstrCost(I, Val, Index); + assert(Cost >= 0 && "TTI should not produce negative costs!"); + return Cost; +} + InstructionCost TargetTransformInfo::getReplicationShuffleCost( Type *EltTy, int ReplicationFactor, int VF, const APInt &DemandedDstElts, TTI::TargetCostKind CostKind) { diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp index b100fbe2b33c40..57423ffbae138e 100644 --- a/llvm/lib/CodeGen/CodeGenPrepare.cpp +++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp @@ -7261,7 +7261,7 @@ class VectorPromoteHelper { // scalar to vector. // The vector chain has to account for the combining cost. InstructionCost ScalarCost = - TTI.getVectorInstrCost(Transition->getOpcode(), PromotedType, Index); + TTI.getVectorInstrCost(Transition, PromotedType, Index); InstructionCost VectorCost = StoreExtractCombineCost; enum TargetTransformInfo::TargetCostKind CostKind = TargetTransformInfo::TCK_RecipThroughput; diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index fcd0df6f1d061a..c4e7135d8b57a5 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -173,6 +173,7 @@ class AArch64TTIImpl : public BasicTTIImplBase { InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h index f2260c31e678d4..eeb30431134210 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h @@ -159,6 +159,7 @@ class GCNTTIImpl final : public BasicTTIImplBase { bool isInlineAsmSourceOfDivergence(const CallInst *CI, ArrayRef Indices = {}) const; + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy, unsigned Index); bool isSourceOfDivergence(const Value *V) const; diff --git a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h index 544292bc4fd9dd..f1a198fd14e499 100644 --- a/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h +++ b/llvm/lib/Target/AMDGPU/R600TargetTransformInfo.h @@ -60,6 +60,7 @@ class R600TTIImpl final : public BasicTTIImplBase { unsigned getMaxInterleaveFactor(unsigned VF); InstructionCost getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *ValTy, unsigned Index); }; diff --git a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h index 5e39e85bac877d..a3aed48f6beb1b 100644 --- a/llvm/lib/Target/ARM/ARMTargetTransformInfo.h +++ b/llvm/lib/Target/ARM/ARMTargetTransformInfo.h @@ -237,6 +237,7 @@ class ARMTTIImpl : public BasicTTIImplBase { TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); diff --git a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h index 7bbaf7ae9cb260..9d263642173a94 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h @@ -151,6 +151,7 @@ class HexagonTTIImpl : public BasicTTIImplBase { TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h index 790eb0b42afae9..9a5b26bd1b9059 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h @@ -123,6 +123,7 @@ class PPCTTIImpl : public BasicTTIImplBase { CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); InstructionCost getMemoryOpCost(unsigned Opcode, Type *Src, diff --git a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h index 33317e799eabb1..0804d0330de49e 100644 --- a/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h +++ b/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h @@ -107,6 +107,7 @@ class SystemZTTIImpl : public BasicTTIImplBase { CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); bool isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue); diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h index fde58a9587b696..10179eff2620dc 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h @@ -67,6 +67,7 @@ class WebAssemblyTTIImpl final : public BasicTTIImplBase { TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, ArrayRef Args = ArrayRef(), const Instruction *CxtI = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h index bd3c3fb1bb2f85..9f83c0461b5619 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.h +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h @@ -146,6 +146,7 @@ class X86TTIImpl : public BasicTTIImplBase { CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, const Instruction *I = nullptr); + using BaseT::getVectorInstrCost; InstructionCost getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index); InstructionCost getScalarizationOverhead(VectorType *Ty, diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index d69d1e3d19f39e..737fcb386764ee 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -5882,8 +5882,7 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, continue; } } - Cost -= TTIRef.getVectorInstrCost(Instruction::ExtractElement, - EE->getVectorOperandType(), Idx); + Cost -= TTIRef.getVectorInstrCost(EE, EE->getVectorOperandType(), Idx); } // Add a cost for subvector extracts/inserts if required. for (const auto &Data : ExtractVectorsTys) { @@ -6116,9 +6115,8 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, for (unsigned I : E->ReuseShuffleIndices) { if (ShuffleOrOp == Instruction::ExtractElement) { auto *EE = cast(VL[I]); - CommonCost -= TTI->getVectorInstrCost(Instruction::ExtractElement, - EE->getVectorOperandType(), - *getExtractIndex(EE)); + CommonCost -= TTI->getVectorInstrCost( + EE, EE->getVectorOperandType(), *getExtractIndex(EE)); } else { CommonCost -= TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, Idx); @@ -6129,9 +6127,8 @@ InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E, for (Value *V : VL) { if (ShuffleOrOp == Instruction::ExtractElement) { auto *EE = cast(V); - CommonCost += TTI->getVectorInstrCost(Instruction::ExtractElement, - EE->getVectorOperandType(), - *getExtractIndex(EE)); + CommonCost += TTI->getVectorInstrCost( + EE, EE->getVectorOperandType(), *getExtractIndex(EE)); } else { --Idx; CommonCost += TTI->getVectorInstrCost(Instruction::ExtractElement, diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp index a38936644bd306..37901d8bce4d25 100644 --- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp +++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp @@ -270,10 +270,8 @@ ExtractElementInst *VectorCombine::getShuffleExtract( Type *VecTy = Ext0->getVectorOperand()->getType(); assert(VecTy == Ext1->getVectorOperand()->getType() && "Need matching types"); - InstructionCost Cost0 = - TTI.getVectorInstrCost(Ext0->getOpcode(), VecTy, Index0); - InstructionCost Cost1 = - TTI.getVectorInstrCost(Ext1->getOpcode(), VecTy, Index1); + InstructionCost Cost0 = TTI.getVectorInstrCost(Ext0, VecTy, Index0); + InstructionCost Cost1 = TTI.getVectorInstrCost(Ext1, VecTy, Index1); // If both costs are invalid no shuffle is needed if (!Cost0.isValid() && !Cost1.isValid()) @@ -337,10 +335,8 @@ bool VectorCombine::isExtractExtractCheap(ExtractElementInst *Ext0, unsigned Ext0Index = Ext0IndexC->getZExtValue(); unsigned Ext1Index = Ext1IndexC->getZExtValue(); - InstructionCost Extract0Cost = - TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy, Ext0Index); - InstructionCost Extract1Cost = - TTI.getVectorInstrCost(Instruction::ExtractElement, VecTy, Ext1Index); + InstructionCost Extract0Cost = TTI.getVectorInstrCost(Ext0, VecTy, Ext0Index); + InstructionCost Extract1Cost = TTI.getVectorInstrCost(Ext1, VecTy, Ext1Index); // A more expensive extract will always be replaced by a splat shuffle. // For example, if Ext0 is more expensive: @@ -754,9 +750,8 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) { if (!VecTy) return false; - InstructionCost OldCost = - TTI.getVectorInstrCost(Ext0->getOpcode(), VecTy, Index0); - OldCost += TTI.getVectorInstrCost(Ext1->getOpcode(), VecTy, Index1); + InstructionCost OldCost = TTI.getVectorInstrCost(Ext0, VecTy, Index0); + OldCost += TTI.getVectorInstrCost(Ext1, VecTy, Index1); OldCost += TTI.getCmpSelInstrCost(CmpOpcode, I0->getType(), CmpInst::makeCmpResultType(I0->getType()), Pred) * @@ -776,7 +771,7 @@ bool VectorCombine::foldExtractedCmps(Instruction &I) { NewCost += TTI.getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc, CmpTy, ShufMask); NewCost += TTI.getArithmeticInstrCost(I.getOpcode(), CmpTy); - NewCost += TTI.getVectorInstrCost(Ext0->getOpcode(), CmpTy, CheapIndex); + NewCost += TTI.getVectorInstrCost(Ext0, CmpTy, CheapIndex); // Aggressively form vector ops if the cost is equal because the transform // may enable further optimization.