Skip to content

Commit

Permalink
[TargetLowering] Add MachineMemOperand::Flags to allowsMemoryAccess t…
Browse files Browse the repository at this point in the history
…ests (PR42123)

As discussed on D62910, we need to check whether particular types of memory access are allowed, not just their alignment/address-space.

This NFC patch adds a MachineMemOperand::Flags argument to allowsMemoryAccess and allowsMisalignedMemoryAccesses, and wires up calls to pass the relevant flags to them.

If people are happy with this approach I can then update X86TargetLowering::allowsMisalignedMemoryAccesses to handle misaligned NT load/stores.

Differential Revision: https://reviews.llvm.org/D63075

llvm-svn: 363179
  • Loading branch information
RKSimon committed Jun 12, 2019
1 parent 5b0e0dd commit 4e0648a
Show file tree
Hide file tree
Showing 28 changed files with 91 additions and 76 deletions.
9 changes: 5 additions & 4 deletions llvm/include/llvm/CodeGen/BasicTTIImpl.h
Expand Up @@ -196,11 +196,12 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
public:
/// \name Scalar TTI Implementations
/// @{
bool allowsMisalignedMemoryAccesses(LLVMContext &Context,
unsigned BitWidth, unsigned AddressSpace,
unsigned Alignment, bool *Fast) const {
bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth,
unsigned AddressSpace, unsigned Alignment,
bool *Fast) const {
EVT E = EVT::getIntegerVT(Context, BitWidth);
return getTLI()->allowsMisalignedMemoryAccesses(E, AddressSpace, Alignment, Fast);
return getTLI()->allowsMisalignedMemoryAccesses(
E, AddressSpace, Alignment, MachineMemOperand::MONone, Fast);
}

bool hasBranchDivergence() { return false; }
Expand Down
16 changes: 9 additions & 7 deletions llvm/include/llvm/CodeGen/TargetLowering.h
Expand Up @@ -1415,20 +1415,22 @@ class TargetLoweringBase {
/// copy/move/set is converted to a sequence of store operations. Its use
/// helps to ensure that such replacements don't generate code that causes an
/// alignment error (trap) on the target machine.
virtual bool allowsMisalignedMemoryAccesses(EVT,
unsigned AddrSpace = 0,
unsigned Align = 1,
bool * /*Fast*/ = nullptr) const {
virtual bool allowsMisalignedMemoryAccesses(
EVT, unsigned AddrSpace = 0, unsigned Align = 1,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool * /*Fast*/ = nullptr) const {
return false;
}

/// Return true if the target supports a memory access of this type for the
/// given address space and alignment. If the access is allowed, the optional
/// final parameter returns if the access is also fast (as defined by the
/// target).
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
unsigned AddrSpace = 0, unsigned Alignment = 1,
bool *Fast = nullptr) const;
bool
allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
unsigned AddrSpace = 0, unsigned Alignment = 1,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *Fast = nullptr) const;

/// Return true if the target supports a memory access of this type for the
/// given MachineMemOperand. If the access is allowed, the optional
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
Expand Up @@ -4638,7 +4638,8 @@ bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST,
// Ensure that this isn't going to produce an unsupported unaligned access.
if (ShAmt &&
!TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT,
LDST->getAddressSpace(), ShAmt / 8))
LDST->getAddressSpace(), ShAmt / 8,
LDST->getMemOperand()->getFlags()))
return false;

// It's not possible to generate a constant of extended or untyped type.
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
Expand Up @@ -238,7 +238,8 @@ TargetLowering::findOptimalMemOpLowering(std::vector<EVT> &MemOps,
// issuing a (or a pair of) unaligned and overlapping load / store.
bool Fast;
if (NumMemOps && AllowOverlap && NewVTSize < Size &&
allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast) &&
allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign,
MachineMemOperand::MONone, &Fast) &&
Fast)
VTSize = Size;
else {
Expand Down
5 changes: 3 additions & 2 deletions llvm/lib/CodeGen/TargetLoweringBase.cpp
Expand Up @@ -1464,6 +1464,7 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
const DataLayout &DL, EVT VT,
unsigned AddrSpace,
unsigned Alignment,
MachineMemOperand::Flags Flags,
bool *Fast) const {
// Check if the specified alignment is sufficient based on the data layout.
// TODO: While using the data layout works in practice, a better solution
Expand All @@ -1479,15 +1480,15 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
}

// This is a misaligned access.
return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Fast);
return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast);
}

bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context,
const DataLayout &DL, EVT VT,
const MachineMemOperand &MMO,
bool *Fast) const {
return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
MMO.getAlignment(), Fast);
MMO.getAlignment(), MMO.getFlags(), Fast);
}

BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const {
Expand Down
14 changes: 8 additions & 6 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Expand Up @@ -1074,10 +1074,9 @@ MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
return MVT::i64;
}

bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned AddrSpace,
unsigned Align,
bool *Fast) const {
bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
bool *Fast) const {
if (Subtarget->requiresStrictAlign())
return false;

Expand Down Expand Up @@ -2843,7 +2842,8 @@ SDValue AArch64TargetLowering::LowerSTORE(SDValue Op,
unsigned AS = StoreNode->getAddressSpace();
unsigned Align = StoreNode->getAlignment();
if (Align < MemVT.getStoreSize() &&
!allowsMisalignedMemoryAccesses(MemVT, AS, Align, nullptr)) {
!allowsMisalignedMemoryAccesses(
MemVT, AS, Align, StoreNode->getMemOperand()->getFlags(), nullptr)) {
return scalarizeVectorStore(StoreNode, DAG);
}

Expand Down Expand Up @@ -8716,7 +8716,9 @@ EVT AArch64TargetLowering::getOptimalMemOpType(
if (memOpAlign(SrcAlign, DstAlign, AlignCheck))
return true;
bool Fast;
return allowsMisalignedMemoryAccesses(VT, 0, 1, &Fast) && Fast;
return allowsMisalignedMemoryAccesses(VT, 0, 1, MachineMemOperand::MONone,
&Fast) &&
Fast;
};

if (CanUseNEON && IsMemset && !IsSmallMemset &&
Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Target/AArch64/AArch64ISelLowering.h
Expand Up @@ -262,9 +262,10 @@ class AArch64TargetLowering : public TargetLowering {

/// Returns true if the target allows unaligned memory accesses of the
/// specified type.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace = 0,
unsigned Align = 1,
bool *Fast = nullptr) const override;
bool allowsMisalignedMemoryAccesses(
EVT VT, unsigned AddrSpace = 0, unsigned Align = 1,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *Fast = nullptr) const override;

/// Provide custom lowering hooks for some operations.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
Expand Down
6 changes: 4 additions & 2 deletions llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
Expand Up @@ -2968,7 +2968,8 @@ SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
// Expand unaligned loads earlier than legalization. Due to visitation order
// problems during legalization, the emitted instructions to pack and unpack
// the bytes again are not eliminated in the case of an unaligned copy.
if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) {
if (!allowsMisalignedMemoryAccesses(
VT, AS, Align, LN->getMemOperand()->getFlags(), &IsFast)) {
if (VT.isVector())
return scalarizeVectorLoad(LN, DAG);

Expand Down Expand Up @@ -3020,7 +3021,8 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N,
// order problems during legalization, the emitted instructions to pack and
// unpack the bytes again are not eliminated in the case of an unaligned
// copy.
if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) {
if (!allowsMisalignedMemoryAccesses(
VT, AS, Align, SN->getMemOperand()->getFlags(), &IsFast)) {
if (VT.isVector())
return scalarizeVectorStore(SN, DAG);

Expand Down
10 changes: 5 additions & 5 deletions llvm/lib/Target/AMDGPU/R600ISelLowering.cpp
Expand Up @@ -1261,7 +1261,8 @@ SDValue R600TargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {

unsigned Align = StoreNode->getAlignment();
if (Align < MemVT.getStoreSize() &&
!allowsMisalignedMemoryAccesses(MemVT, AS, Align, nullptr)) {
!allowsMisalignedMemoryAccesses(
MemVT, AS, Align, StoreNode->getMemOperand()->getFlags(), nullptr)) {
return expandUnalignedStore(StoreNode, DAG);
}

Expand Down Expand Up @@ -1663,10 +1664,9 @@ bool R600TargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
return true;
}

bool R600TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned AddrSpace,
unsigned Align,
bool *IsFast) const {
bool R600TargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
bool *IsFast) const {
if (IsFast)
*IsFast = false;

Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Target/AMDGPU/R600ISelLowering.h
Expand Up @@ -49,9 +49,10 @@ class R600TargetLowering final : public AMDGPUTargetLowering {
bool canMergeStoresTo(unsigned AS, EVT MemVT,
const SelectionDAG &DAG) const override;

bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS,
unsigned Align,
bool *IsFast) const override;
bool allowsMisalignedMemoryAccesses(
EVT VT, unsigned AS, unsigned Align,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *IsFast = nullptr) const override;

private:
unsigned Gen;
Expand Down
7 changes: 3 additions & 4 deletions llvm/lib/Target/AMDGPU/SIISelLowering.cpp
Expand Up @@ -1157,10 +1157,9 @@ bool SITargetLowering::canMergeStoresTo(unsigned AS, EVT MemVT,
return true;
}

bool SITargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned AddrSpace,
unsigned Align,
bool *IsFast) const {
bool SITargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned AddrSpace, unsigned Align, MachineMemOperand::Flags Flags,
bool *IsFast) const {
if (IsFast)
*IsFast = false;

Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Target/AMDGPU/SIISelLowering.h
Expand Up @@ -233,9 +233,10 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool canMergeStoresTo(unsigned AS, EVT MemVT,
const SelectionDAG &DAG) const override;

bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS,
unsigned Align,
bool *IsFast) const override;
bool allowsMisalignedMemoryAccesses(
EVT VT, unsigned AS, unsigned Align,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *IsFast = nullptr) const override;

EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign,
unsigned SrcAlign, bool IsMemset,
Expand Down
11 changes: 7 additions & 4 deletions llvm/lib/Target/ARM/ARMISelLowering.cpp
Expand Up @@ -13043,9 +13043,9 @@ bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
return (VT == MVT::f32) && (Opc == ISD::LOAD || Opc == ISD::STORE);
}

bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned,
bool ARMTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned,
unsigned,
MachineMemOperand::Flags,
bool *Fast) const {
// Depends what it gets converted into if the type is weird.
if (!VT.isSimple())
Expand Down Expand Up @@ -13099,11 +13099,14 @@ EVT ARMTargetLowering::getOptimalMemOpType(
bool Fast;
if (Size >= 16 &&
(memOpAlign(SrcAlign, DstAlign, 16) ||
(allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1, &Fast) && Fast))) {
(allowsMisalignedMemoryAccesses(MVT::v2f64, 0, 1,
MachineMemOperand::MONone, &Fast) &&
Fast))) {
return MVT::v2f64;
} else if (Size >= 8 &&
(memOpAlign(SrcAlign, DstAlign, 8) ||
(allowsMisalignedMemoryAccesses(MVT::f64, 0, 1, &Fast) &&
(allowsMisalignedMemoryAccesses(
MVT::f64, 0, 1, MachineMemOperand::MONone, &Fast) &&
Fast))) {
return MVT::f64;
}
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/ARM/ARMISelLowering.h
Expand Up @@ -321,6 +321,7 @@ class VectorType;
/// is "fast" by reference in the second argument.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,
unsigned Align,
MachineMemOperand::Flags Flags,
bool *Fast) const override;

EVT getOptimalMemOpType(uint64_t Size,
Expand Down
5 changes: 3 additions & 2 deletions llvm/lib/Target/Hexagon/HexagonISelLowering.cpp
Expand Up @@ -3064,8 +3064,9 @@ EVT HexagonTargetLowering::getOptimalMemOpType(uint64_t Size,
return MVT::Other;
}

bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned AS, unsigned Align, bool *Fast) const {
bool HexagonTargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned AS, unsigned Align, MachineMemOperand::Flags Flags,
bool *Fast) const {
if (Fast)
*Fast = false;
return Subtarget.isHVXVectorType(VT.getSimpleVT());
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/Hexagon/HexagonISelLowering.h
Expand Up @@ -298,7 +298,7 @@ namespace HexagonISD {
const AttributeList &FuncAttributes) const override;

bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,
unsigned Align, bool *Fast) const override;
unsigned Align, MachineMemOperand::Flags Flags, bool *Fast) const override;

/// Returns relocation base for the given PIC jumptable.
SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG)
Expand Down
7 changes: 2 additions & 5 deletions llvm/lib/Target/Mips/Mips16ISelLowering.cpp
Expand Up @@ -155,11 +155,8 @@ llvm::createMips16TargetLowering(const MipsTargetMachine &TM,
return new Mips16TargetLowering(TM, STI);
}

bool
Mips16TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned,
unsigned,
bool *Fast) const {
bool Mips16TargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const {
return false;
}

Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/Mips/Mips16ISelLowering.h
Expand Up @@ -23,6 +23,7 @@ namespace llvm {

bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace,
unsigned Align,
MachineMemOperand::Flags Flags,
bool *Fast) const override;

MachineBasicBlock *
Expand Down
7 changes: 2 additions & 5 deletions llvm/lib/Target/Mips/MipsSEISelLowering.cpp
Expand Up @@ -419,11 +419,8 @@ SDValue MipsSETargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
Op->getOperand(2));
}

bool
MipsSETargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned,
unsigned,
bool *Fast) const {
bool MipsSETargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const {
MVT::SimpleValueType SVT = VT.getSimpleVT().SimpleTy;

if (Subtarget.systemSupportsUnalignedAccess()) {
Expand Down
7 changes: 4 additions & 3 deletions llvm/lib/Target/Mips/MipsSEISelLowering.h
Expand Up @@ -40,9 +40,10 @@ class TargetRegisterClass;
void addMSAFloatType(MVT::SimpleValueType Ty,
const TargetRegisterClass *RC);

bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS = 0,
unsigned Align = 1,
bool *Fast = nullptr) const override;
bool allowsMisalignedMemoryAccesses(
EVT VT, unsigned AS = 0, unsigned Align = 1,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *Fast = nullptr) const override;

SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;

Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/PowerPC/PPCISelLowering.cpp
Expand Up @@ -14566,6 +14566,7 @@ bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned,
unsigned,
MachineMemOperand::Flags,
bool *Fast) const {
if (DisablePPCUnaligned)
return false;
Expand Down
8 changes: 4 additions & 4 deletions llvm/lib/Target/PowerPC/PPCISelLowering.h
Expand Up @@ -846,10 +846,10 @@ namespace llvm {

/// Is unaligned memory access allowed for the given type, and is it fast
/// relative to software emulation.
bool allowsMisalignedMemoryAccesses(EVT VT,
unsigned AddrSpace,
unsigned Align = 1,
bool *Fast = nullptr) const override;
bool allowsMisalignedMemoryAccesses(
EVT VT, unsigned AddrSpace, unsigned Align = 1,
MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
bool *Fast = nullptr) const override;

/// isFMAFasterThanFMulAndFAdd - Return true if an FMA operation is faster
/// than a pair of fmul and fadd instructions. fmuladd intrinsics will be
Expand Down
6 changes: 2 additions & 4 deletions llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
Expand Up @@ -761,10 +761,8 @@ bool SystemZTargetLowering::isLegalAddImmediate(int64_t Imm) const {
return isUInt<32>(Imm) || isUInt<32>(-Imm);
}

bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
unsigned,
unsigned,
bool *Fast) const {
bool SystemZTargetLowering::allowsMisalignedMemoryAccesses(
EVT VT, unsigned, unsigned, MachineMemOperand::Flags, bool *Fast) const {
// Unaligned accesses should never be slower than the expanded version.
// We check specifically for aligned accesses in the few cases where
// they are required.
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/SystemZ/SystemZISelLowering.h
Expand Up @@ -409,6 +409,7 @@ class SystemZTargetLowering : public TargetLowering {
Instruction *I = nullptr) const override;
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS,
unsigned Align,
MachineMemOperand::Flags Flags,
bool *Fast) const override;
bool isTruncateFree(Type *, Type *) const override;
bool isTruncateFree(EVT, EVT) const override;
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
Expand Up @@ -530,7 +530,8 @@ bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
}

bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
EVT /*VT*/, unsigned /*AddrSpace*/, unsigned /*Align*/, bool *Fast) const {
EVT /*VT*/, unsigned /*AddrSpace*/, unsigned /*Align*/,
MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
// WebAssembly supports unaligned accesses, though it should be declared
// with the p2align attribute on loads and stores which do so, and there
// may be a performance impact. We tell LLVM they're "fast" because
Expand Down
1 change: 1 addition & 0 deletions llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h
Expand Up @@ -60,6 +60,7 @@ class WebAssemblyTargetLowering final : public TargetLowering {
unsigned AS,
Instruction *I = nullptr) const override;
bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace, unsigned Align,
MachineMemOperand::Flags Flags,
bool *Fast) const override;
bool isIntDivCheap(EVT VT, AttributeList Attr) const override;

Expand Down

0 comments on commit 4e0648a

Please sign in to comment.