Skip to content

Commit

Permalink
Revert "Recommit "[SLP] Fix lookahead operand reordering for splat lo…
Browse files Browse the repository at this point in the history 
…ads." attempt 2, fixed assertion crash."

This reverts commit 27bd8f9.

Causes crashes, see comments in D121973
  • Loading branch information
@aeubanks
aeubanks committed Mar 23, 2022
1 parent f858fba commit e6ead19
Show file tree
Hide file tree
Showing 23 changed files with 96 additions and 237 deletions.
25 changes: 5 additions & 20 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
View file
Expand Up @@ -658,10 +658,6 @@ class TargetTransformInfo {
/// Return true if the target supports nontemporal load.
bool isLegalNTLoad(Type *DataType, Align Alignment) const;

/// \Returns true if the target supports broadcasting a load to a vector of
/// type <NumElements x ElementTy>.
bool isLegalBroadcastLoad(Type *ElementTy, unsigned NumElements) const;

/// Return true if the target supports masked scatter.
bool isLegalMaskedScatter(Type *DataType, Align Alignment) const;
/// Return true if the target supports masked gather.
Expand Down Expand Up @@ -1048,14 +1044,11 @@ class TargetTransformInfo {
/// The exact mask may be passed as Mask, or else the array will be empty.
/// The index and subtype parameters are used by the subvector insertion and
/// extraction shuffle kinds to show the insert/extract point and the type of
/// the subvector being inserted/extracted. The operands of the shuffle can be
/// passed through \p Args, which helps improve the cost estimation in some
/// cases, like in broadcast loads.
/// the subvector being inserted/extracted.
/// NOTE: For subvector extractions Tp represents the source type.
InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask = None, int Index = 0,
VectorType *SubTp = nullptr,
ArrayRef<Value *> Args = None) const;
VectorType *SubTp = nullptr) const;

/// Represents a hint about the context in which a cast is used.
///
Expand Down Expand Up @@ -1556,8 +1549,6 @@ class TargetTransformInfo::Concept {
virtual bool isLegalMaskedLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalNTStore(Type *DataType, Align Alignment) = 0;
virtual bool isLegalNTLoad(Type *DataType, Align Alignment) = 0;
virtual bool isLegalBroadcastLoad(Type *ElementTy,
unsigned NumElements) const = 0;
virtual bool isLegalMaskedScatter(Type *DataType, Align Alignment) = 0;
virtual bool isLegalMaskedGather(Type *DataType, Align Alignment) = 0;
virtual bool forceScalarizeMaskedGather(VectorType *DataType,
Expand Down Expand Up @@ -1668,8 +1659,7 @@ class TargetTransformInfo::Concept {
ArrayRef<const Value *> Args, const Instruction *CxtI = nullptr) = 0;
virtual InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args) = 0;
VectorType *SubTp) = 0;
virtual InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst,
Type *Src, CastContextHint CCH,
TTI::TargetCostKind CostKind,
Expand Down Expand Up @@ -1962,10 +1952,6 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
bool isLegalNTLoad(Type *DataType, Align Alignment) override {
return Impl.isLegalNTLoad(DataType, Alignment);
}
bool isLegalBroadcastLoad(Type *ElementTy,
unsigned NumElements) const override {
return Impl.isLegalBroadcastLoad(ElementTy, NumElements);
}
bool isLegalMaskedScatter(Type *DataType, Align Alignment) override {
return Impl.isLegalMaskedScatter(DataType, Alignment);
}
Expand Down Expand Up @@ -2193,9 +2179,8 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
}
InstructionCost getShuffleCost(ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args) override {
return Impl.getShuffleCost(Kind, Tp, Mask, Index, SubTp, Args);
VectorType *SubTp) override {
return Impl.getShuffleCost(Kind, Tp, Mask, Index, SubTp);
}
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
CastContextHint CCH,
Expand Down
7 changes: 1 addition & 6 deletions llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View file
Expand Up @@ -256,10 +256,6 @@ class TargetTransformInfoImplBase {
return Alignment >= DataSize && isPowerOf2_32(DataSize);
}

bool isLegalBroadcastLoad(Type *ElementTy, unsigned NumElements) const {
return false;
}

bool isLegalMaskedScatter(Type *DataType, Align Alignment) const {
return false;
}
Expand Down Expand Up @@ -492,8 +488,7 @@ class TargetTransformInfoImplBase {

InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Ty,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None) const {
VectorType *SubTp) const {
return 1;
}

Expand Down
3 changes: 1 addition & 2 deletions llvm/include/llvm/CodeGen/BasicTTIImpl.h
View file
Expand Up @@ -871,8 +871,7 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {

InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None) {
VectorType *SubTp) {

switch (improveShuffleKindFromMask(Kind, Mask)) {
case TTI::SK_Broadcast:
Expand Down
16 changes: 6 additions & 10 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
View file
Expand Up @@ -396,11 +396,6 @@ bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const {
return TTIImpl->isLegalNTLoad(DataType, Alignment);
}

bool TargetTransformInfo::isLegalBroadcastLoad(Type *ElementTy,
unsigned NumElements) const {
return TTIImpl->isLegalBroadcastLoad(ElementTy, NumElements);
}

bool TargetTransformInfo::isLegalMaskedGather(Type *DataType,
Align Alignment) const {
return TTIImpl->isLegalMaskedGather(DataType, Alignment);
Expand Down Expand Up @@ -745,11 +740,12 @@ InstructionCost TargetTransformInfo::getArithmeticInstrCost(
return Cost;
}

InstructionCost TargetTransformInfo::getShuffleCost(
ShuffleKind Kind, VectorType *Ty, ArrayRef<int> Mask, int Index,
VectorType *SubTp, ArrayRef<Value *> Args) const {
InstructionCost Cost =
TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp, Args);
InstructionCost TargetTransformInfo::getShuffleCost(ShuffleKind Kind,
VectorType *Ty,
ArrayRef<int> Mask,
int Index,
VectorType *SubTp) const {
InstructionCost Cost = TTIImpl->getShuffleCost(Kind, Ty, Mask, Index, SubTp);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
View file
Expand Up @@ -2604,8 +2604,7 @@ InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) {
InstructionCost AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args) {
VectorType *SubTp) {
Kind = improveShuffleKindFromMask(Kind, Mask);
if (Kind == TTI::SK_Broadcast || Kind == TTI::SK_Transpose ||
Kind == TTI::SK_Select || Kind == TTI::SK_PermuteSingleSrc ||
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
View file
Expand Up @@ -330,8 +330,7 @@ class AArch64TTIImpl : public BasicTTIImplBase<AArch64TTIImpl> {

InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None);
VectorType *SubTp);
/// @}
};

Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
View file
Expand Up @@ -1042,8 +1042,7 @@ Value *GCNTTIImpl::rewriteIntrinsicWithAddressSpace(IntrinsicInst *II,

InstructionCost GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
VectorType *VT, ArrayRef<int> Mask,
int Index, VectorType *SubTp,
ArrayRef<Value *> Args) {
int Index, VectorType *SubTp) {
Kind = improveShuffleKindFromMask(Kind, Mask);
if (ST->hasVOP3PInsts()) {
if (cast<FixedVectorType>(VT)->getNumElements() == 2 &&
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
View file
Expand Up @@ -201,8 +201,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {

InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None);
VectorType *SubTp);

bool areInlineCompatible(const Function *Caller,
const Function *Callee) const;
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
View file
Expand Up @@ -1202,8 +1202,7 @@ InstructionCost ARMTTIImpl::getMemcpyCost(const Instruction *I) {

InstructionCost ARMTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
VectorType *Tp, ArrayRef<int> Mask,
int Index, VectorType *SubTp,
ArrayRef<Value *> Args) {
int Index, VectorType *SubTp) {
Kind = improveShuffleKindFromMask(Kind, Mask);
if (ST->hasNEON()) {
if (Kind == TTI::SK_Broadcast) {
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/ARM/ARMTargetTransformInfo.h
View file
Expand Up @@ -213,8 +213,7 @@ class ARMTTIImpl : public BasicTTIImplBase<ARMTTIImpl> {

InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None);
VectorType *SubTp);

bool preferInLoopReduction(unsigned Opcode, Type *Ty,
TTI::ReductionFlags Flags) const;
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.cpp
View file
Expand Up @@ -223,8 +223,7 @@ HexagonTTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src,

InstructionCost HexagonTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
ArrayRef<int> Mask, int Index,
Type *SubTp,
ArrayRef<Value *> Args) {
Type *SubTp) {
return 1;
}

Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/Hexagon/HexagonTargetTransformInfo.h
View file
Expand Up @@ -125,8 +125,7 @@ class HexagonTTIImpl : public BasicTTIImplBase<HexagonTTIImpl> {
Align Alignment, unsigned AddressSpace,
TTI::TargetCostKind CostKind);
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
ArrayRef<int> Mask, int Index, Type *SubTp,
ArrayRef<Value *> Args = None);
ArrayRef<int> Mask, int Index, Type *SubTp);
InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
const Value *Ptr, bool VariableMask,
Align Alignment,
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp
View file
Expand Up @@ -1015,8 +1015,7 @@ InstructionCost PPCTTIImpl::getArithmeticInstrCost(

InstructionCost PPCTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
ArrayRef<int> Mask, int Index,
Type *SubTp,
ArrayRef<Value *> Args) {
Type *SubTp) {

InstructionCost CostFactor =
vectorCostAdjustmentFactor(Instruction::ShuffleVector, Tp, nullptr);
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/PowerPC/PPCTargetTransformInfo.h
View file
Expand Up @@ -111,8 +111,7 @@ class PPCTTIImpl : public BasicTTIImplBase<PPCTTIImpl> {
ArrayRef<const Value *> Args = ArrayRef<const Value *>(),
const Instruction *CxtI = nullptr);
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, Type *Tp,
ArrayRef<int> Mask, int Index, Type *SubTp,
ArrayRef<Value *> Args = None);
ArrayRef<int> Mask, int Index, Type *SubTp);
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
View file
Expand Up @@ -175,8 +175,7 @@ InstructionCost RISCVTTIImpl::getSpliceCost(VectorType *Tp, int Index) {

InstructionCost RISCVTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
VectorType *Tp, ArrayRef<int> Mask,
int Index, VectorType *SubTp,
ArrayRef<Value *> Args) {
int Index, VectorType *SubTp) {
if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Tp))
return getSpliceCost(Tp, Index);
return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
View file
Expand Up @@ -80,8 +80,7 @@ class RISCVTTIImpl : public BasicTTIImplBase<RISCVTTIImpl> {
InstructionCost getSpliceCost(VectorType *Tp, int Index);
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None);
VectorType *SubTp);

InstructionCost getGatherScatterOpCost(unsigned Opcode, Type *DataTy,
const Value *Ptr, bool VariableMask,
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp
View file
Expand Up @@ -559,8 +559,7 @@ InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
InstructionCost SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args) {
VectorType *SubTp) {
Kind = improveShuffleKindFromMask(Kind, Mask);
if (ST->hasVector()) {
unsigned NumVectors = getNumVectorRegs(Tp);
Expand Down
3 changes: 1 addition & 2 deletions llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.h
View file
Expand Up @@ -92,8 +92,7 @@ class SystemZTTIImpl : public BasicTTIImplBase<SystemZTTIImpl> {
const Instruction *CxtI = nullptr);
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None);
VectorType *SubTp);
unsigned getVectorTruncCost(Type *SrcTy, Type *DstTy);
unsigned getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy);
unsigned getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
Expand Down
30 changes: 2 additions & 28 deletions llvm/lib/Target/X86/X86TargetTransformInfo.cpp
View file
Expand Up @@ -1085,8 +1085,7 @@ InstructionCost X86TTIImpl::getArithmeticInstrCost(
InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
VectorType *BaseTp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args) {
VectorType *SubTp) {
// 64-bit packed float vectors (v2f32) are widened to type v4f32.
// 64-bit packed integer vectors (v2i32) are widened to type v4i32.
std::pair<InstructionCost, MVT> LT = TLI->getTypeLegalizationCost(DL, BaseTp);
Expand Down Expand Up @@ -1546,27 +1545,9 @@ InstructionCost X86TTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
{ TTI::SK_PermuteTwoSrc, MVT::v16i8, 13 }, // blend+permute
};

static const CostTblEntry SSE3BroadcastLoadTbl[] = {
{TTI::SK_Broadcast, MVT::v2f64, 0}, // broadcast handled by movddup
};

if (ST->hasSSE2()) {
bool IsLoad = !Args.empty() && llvm::all_of(Args, [](const Value *V) {
return isa<LoadInst>(V);
});
if (ST->hasSSE3() && IsLoad)
if (const auto *Entry =
CostTableLookup(SSE3BroadcastLoadTbl, Kind, LT.second)) {
assert(isLegalBroadcastLoad(
BaseTp->getElementType(),
cast<FixedVectorType>(BaseTp)->getNumElements()) &&
"Table entry missing from isLegalBroadcastLoad()");
return LT.first * Entry->Cost;
}

if (ST->hasSSE2())
if (const auto *Entry = CostTableLookup(SSE2ShuffleTbl, Kind, LT.second))
return LT.first * Entry->Cost;
}

static const CostTblEntry SSE1ShuffleTbl[] = {
{ TTI::SK_Broadcast, MVT::v4f32, 1 }, // shufps
Expand Down Expand Up @@ -5137,13 +5118,6 @@ bool X86TTIImpl::isLegalNTStore(Type *DataType, Align Alignment) {
return true;
}

bool X86TTIImpl::isLegalBroadcastLoad(Type *ElementTy,
unsigned NumElements) const {
// movddup
return ST->hasSSE3() && NumElements == 2 &&
ElementTy == Type::getDoubleTy(ElementTy->getContext());
}

bool X86TTIImpl::isLegalMaskedExpandLoad(Type *DataTy) {
if (!isa<VectorType>(DataTy))
return false;
Expand Down
4 changes: 1 addition & 3 deletions llvm/lib/Target/X86/X86TargetTransformInfo.h
View file
Expand Up @@ -131,8 +131,7 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
const Instruction *CxtI = nullptr);
InstructionCost getShuffleCost(TTI::ShuffleKind Kind, VectorType *Tp,
ArrayRef<int> Mask, int Index,
VectorType *SubTp,
ArrayRef<Value *> Args = None);
VectorType *SubTp);
InstructionCost getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
Expand Down Expand Up @@ -227,7 +226,6 @@ class X86TTIImpl : public BasicTTIImplBase<X86TTIImpl> {
bool isLegalMaskedStore(Type *DataType, Align Alignment);
bool isLegalNTLoad(Type *DataType, Align Alignment);
bool isLegalNTStore(Type *DataType, Align Alignment);
bool isLegalBroadcastLoad(Type *ElementTy, unsigned NumElements) const;
bool forceScalarizeMaskedGather(VectorType *VTy, Align Alignment);
bool forceScalarizeMaskedScatter(VectorType *VTy, Align Alignment) {
return forceScalarizeMaskedGather(VTy, Alignment);
Expand Down

0 comments on commit e6ead19

Please sign in to comment.