Skip to content

[TTI] Use MemIntrinsicCostAttributes for getExpandCompressMemoryOpCost #168677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+46 −33
Open

[TTI] Use MemIntrinsicCostAttributes for getExpandCompressMemoryOpCost #168677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 18 additions & 10 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,12 +125,20 @@ struct HardwareLoopInfo {

/// Information for memory intrinsic cost model.
class MemIntrinsicCostAttributes {
/// Optional context instruction, if one exists, e.g. the
/// load/store to transform to the intrinsic.
const Instruction *I = nullptr;

/// Vector type of the data to be loaded or stored.
Type *DataTy = nullptr;

/// ID of the memory intrinsic.
Intrinsic::ID IID;

/// True when the memory access is predicated with a mask
/// that is not a compile-time constant.
bool VariableMask = true;
Copy link
Contributor

@lukel97 lukel97 Nov 19, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this default to false?

Copy link
Contributor Author

@arcbbb arcbbb Nov 20, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had it as false initially, but found that the existing paths treat it as true when unspecified.

  • getMaskedMemoryOpCost forwarding to getCommonMaskedMemoryOpCost
  • experimental_vp_strided_* in getTypeBasedIntrinsicInstrCost

so I follow that and do the same.


/// Address space of the pointer.
unsigned AddressSpace = 0;

Expand All @@ -143,8 +151,16 @@ class MemIntrinsicCostAttributes {
: DataTy(DataTy), IID(Id), AddressSpace(AddressSpace),
Alignment(Alignment) {}

LLVM_ABI MemIntrinsicCostAttributes(Intrinsic::ID Id, Type *DataTy,
bool VariableMask, Align Alignment,
const Instruction *I = nullptr)
: I(I), DataTy(DataTy), IID(Id), VariableMask(VariableMask),
Alignment(Alignment) {}

Intrinsic::ID getID() const { return IID; }
const Instruction *getInst() const { return I; }
Type *getDataType() const { return DataTy; }
bool getVariableMask() const { return VariableMask; }
unsigned getAddressSpace() const { return AddressSpace; }
Align getAlignment() const { return Alignment; }
};
Expand Down Expand Up @@ -1600,17 +1616,9 @@ class TargetTransformInfo {
const Instruction *I = nullptr) const;

/// \return The cost of Expand Load or Compress Store operation
/// \p Opcode - is a type of memory access Load or Store
/// \p Src - a vector type of the data to be loaded or stored
/// \p VariableMask - true when the memory access is predicated with a mask
/// that is not a compile-time constant
/// \p Alignment - alignment of single element
/// \p I - the optional original context instruction, if one exists, e.g. the
/// load/store to transform or the call to the gather/scatter intrinsic
LLVM_ABI InstructionCost getExpandCompressMemoryOpCost(
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput,
const Instruction *I = nullptr) const;
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;

/// \return The cost of strided memory operations.
/// \p Opcode - is a type of memory access Load or Store
Expand Down
6 changes: 3 additions & 3 deletions llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -855,9 +855,9 @@ class TargetTransformInfoImplBase {
return 1;
}

virtual InstructionCost getExpandCompressMemoryOpCost(
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind, const Instruction *I = nullptr) const {
virtual InstructionCost
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
return 1;
}

Expand Down
20 changes: 11 additions & 9 deletions llvm/include/llvm/CodeGen/BasicTTIImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1580,10 +1580,14 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
}

InstructionCost
getExpandCompressMemoryOpCost(unsigned Opcode, Type *DataTy,
bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override {
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override {
unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
? Instruction::Load
: Instruction::Store;
Type *DataTy = MICA.getDataType();
bool VariableMask = MICA.getVariableMask();
Align Alignment = MICA.getAlignment();
// Treat expand load/compress store as gather/scatter operation.
// TODO: implement more precise cost estimation for these intrinsics.
return getCommonMaskedMemoryOpCost(Opcode, DataTy, Alignment, VariableMask,
Expand Down Expand Up @@ -1964,15 +1968,13 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
const Value *Mask = Args[2];
Align Alignment = I->getParamAlign(1).valueOrOne();
return thisT()->getExpandCompressMemoryOpCost(
Instruction::Store, Data->getType(), !isa<Constant>(Mask), Alignment,
CostKind, I);
{IID, Data->getType(), !isa<Constant>(Mask), Alignment, I}, CostKind);
}
case Intrinsic::masked_expandload: {
const Value *Mask = Args[1];
Align Alignment = I->getParamAlign(0).valueOrOne();
return thisT()->getExpandCompressMemoryOpCost(Instruction::Load, RetTy,
!isa<Constant>(Mask),
Alignment, CostKind, I);
return thisT()->getExpandCompressMemoryOpCost(
{IID, RetTy, !isa<Constant>(Mask), Alignment, I}, CostKind);
}
case Intrinsic::experimental_vp_strided_store: {
const Value *Data = Args[0];
Expand Down
7 changes: 3 additions & 4 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1201,10 +1201,9 @@ InstructionCost TargetTransformInfo::getGatherScatterOpCost(
}

InstructionCost TargetTransformInfo::getExpandCompressMemoryOpCost(
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind, const Instruction *I) const {
InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(
Opcode, DataTy, VariableMask, Alignment, CostKind, I);
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
InstructionCost Cost = TTIImpl->getExpandCompressMemoryOpCost(MICA, CostKind);
assert(Cost >= 0 && "TTI should not produce negative costs!");
return Cost;
}
Expand Down
13 changes: 9 additions & 4 deletions llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1146,15 +1146,20 @@ InstructionCost RISCVTTIImpl::getGatherScatterOpCost(
}

InstructionCost RISCVTTIImpl::getExpandCompressMemoryOpCost(
unsigned Opcode, Type *DataTy, bool VariableMask, Align Alignment,
TTI::TargetCostKind CostKind, const Instruction *I) const {
const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const {
unsigned Opcode = MICA.getID() == Intrinsic::masked_expandload
? Instruction::Load
: Instruction::Store;
Type *DataTy = MICA.getDataType();
bool VariableMask = MICA.getVariableMask();
Align Alignment = MICA.getAlignment();
bool IsLegal = (Opcode == Instruction::Store &&
isLegalMaskedCompressStore(DataTy, Alignment)) ||
(Opcode == Instruction::Load &&
isLegalMaskedExpandLoad(DataTy, Alignment));
if (!IsLegal || CostKind != TTI::TCK_RecipThroughput)
return BaseT::getExpandCompressMemoryOpCost(Opcode, DataTy, VariableMask,
Alignment, CostKind, I);
return BaseT::getExpandCompressMemoryOpCost(MICA, CostKind);
// Example compressstore sequence:
// vsetivli zero, 8, e32, m2, ta, ma (ignored)
// vcompress.vm v10, v8, v0
Expand Down
5 changes: 2 additions & 3 deletions llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,8 @@ class RISCVTTIImpl final : public BasicTTIImplBase<RISCVTTIImpl> {
const Instruction *I) const override;

InstructionCost
getExpandCompressMemoryOpCost(unsigned Opcode, Type *Src, bool VariableMask,
Align Alignment, TTI::TargetCostKind CostKind,
const Instruction *I = nullptr) const override;
getExpandCompressMemoryOpCost(const MemIntrinsicCostAttributes &MICA,
TTI::TargetCostKind CostKind) const override;

InstructionCost getStridedMemoryOpCost(unsigned Opcode, Type *DataTy,
const Value *Ptr, bool VariableMask,
Expand Down