diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 59791738fdf6c..cde9ff735af86 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -122,6 +122,7 @@ struct LegalityQuery { struct MemDesc { uint64_t SizeInBits; + uint64_t AlignInBits; AtomicOrdering Ordering; }; @@ -164,13 +165,23 @@ using LegalizeMutation = std::function(const LegalityQuery &)>; namespace LegalityPredicates { -struct TypePairAndMemSize { +struct TypePairAndMemDesc { LLT Type0; LLT Type1; uint64_t MemSize; + uint64_t Align; - bool operator==(const TypePairAndMemSize &Other) const { + bool operator==(const TypePairAndMemDesc &Other) const { return Type0 == Other.Type0 && Type1 == Other.Type1 && + Align == Other.Align && + MemSize == Other.MemSize; + } + + /// \returns true if this memory access is legal with for the acecss described + /// by \p Other (The alignment is sufficient for the size and result type). + bool isCompatible(const TypePairAndMemDesc &Other) const { + return Type0 == Other.Type0 && Type1 == Other.Type1 && + Align >= Other.Align && MemSize == Other.MemSize; } }; @@ -199,9 +210,9 @@ typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, std::initializer_list> TypesInit); /// True iff the given types for the given pair of type indexes is one of the /// specified type pairs. -LegalityPredicate typePairAndMemSizeInSet( +LegalityPredicate typePairAndMemDescInSet( unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, - std::initializer_list TypesAndMemSizeInit); + std::initializer_list TypesAndMemDescInit); /// True iff the specified type index is a scalar. LegalityPredicate isScalar(unsigned TypeIdx); /// True iff the specified type index is a vector. @@ -455,13 +466,13 @@ class LegalizeRuleSet { return actionFor(LegalizeAction::Legal, Types); } /// The instruction is legal when type indexes 0 and 1 along with the memory - /// size is any type and size tuple in the given list. - LegalizeRuleSet &legalForTypesWithMemSize( - std::initializer_list - TypesAndMemSize) { + /// size and minimum alignment is any type and size tuple in the given list. + LegalizeRuleSet &legalForTypesWithMemDesc( + std::initializer_list + TypesAndMemDesc) { return actionIf(LegalizeAction::Legal, - LegalityPredicates::typePairAndMemSizeInSet( - typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemSize)); + LegalityPredicates::typePairAndMemDescInSet( + typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); } /// The instruction is legal when type indexes 0 and 1 are both in the given /// list. That is, the type pair is in the cartesian product of the list. diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp index 07e0cb662b582..601d50e9806fd 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp @@ -38,15 +38,19 @@ LegalityPredicate LegalityPredicates::typePairInSet( }; } -LegalityPredicate LegalityPredicates::typePairAndMemSizeInSet( +LegalityPredicate LegalityPredicates::typePairAndMemDescInSet( unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, - std::initializer_list TypesAndMemSizeInit) { - SmallVector TypesAndMemSize = TypesAndMemSizeInit; - return [=](const LegalityQuery &Query) { - TypePairAndMemSize Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1], - Query.MMODescrs[MMOIdx].SizeInBits}; - return std::find(TypesAndMemSize.begin(), TypesAndMemSize.end(), Match) != - TypesAndMemSize.end(); + std::initializer_list TypesAndMemDescInit) { + SmallVector TypesAndMemDesc = TypesAndMemDescInit; + return [=](const LegalityQuery &Query) { + TypePairAndMemDesc Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1], + Query.MMODescrs[MMOIdx].SizeInBits, + Query.MMODescrs[MMOIdx].AlignInBits}; + return std::find_if( + TypesAndMemDesc.begin(), TypesAndMemDesc.end(), + [=](const TypePairAndMemDesc &Entry) ->bool { + return Match.isCompatible(Entry); + }) != TypesAndMemDesc.end(); }; } diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp index f2fc2e78a9eff..e17993987f0a1 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -423,8 +423,9 @@ LegalizerInfo::getAction(const MachineInstr &MI, SmallVector MemDescrs; for (const auto &MMO : MI.memoperands()) - MemDescrs.push_back( - {MMO->getSize() /* in bytes */ * 8, MMO->getOrdering()}); + MemDescrs.push_back({8 * MMO->getSize() /* in bits */, + 8 * MMO->getAlignment(), + MMO->getOrdering()}); return getAction({MI.getOpcode(), Types, MemDescrs}); } diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp index 34ce38896dacf..3c57af84f03a6 100644 --- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -192,12 +192,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { .widenScalarToNextPow2(0); getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}) - .legalForTypesWithMemSize({{s32, p0, 8}, - {s32, p0, 16}, - {s32, p0, 32}, - {s64, p0, 64}, - {p0, p0, 64}, - {v2s32, p0, 64}}) + .legalForTypesWithMemDesc({{s32, p0, 8, 8}, + {s32, p0, 16, 8}, + {s32, p0, 32, 8}, + {s64, p0, 64, 8}, + {p0, p0, 64, 8}, + {v2s32, p0, 64, 8}}) .clampScalar(0, s32, s64) .widenScalarToNextPow2(0) // TODO: We could support sum-of-pow2's but the lowering code doesn't know @@ -207,15 +207,15 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { .lower(); getActionDefinitionsBuilder(G_LOAD) - .legalForTypesWithMemSize({{s8, p0, 8}, - {s16, p0, 16}, - {s32, p0, 32}, - {s64, p0, 64}, - {p0, p0, 64}, - {v2s32, p0, 64}}) + .legalForTypesWithMemDesc({{s8, p0, 8, 8}, + {s16, p0, 16, 8}, + {s32, p0, 32, 8}, + {s64, p0, 64, 8}, + {p0, p0, 64, 8}, + {v2s32, p0, 64, 8}}) // These extends are also legal - .legalForTypesWithMemSize({{s32, p0, 8}, - {s32, p0, 16}}) + .legalForTypesWithMemDesc({{s32, p0, 8, 8}, + {s32, p0, 16, 8}}) .clampScalar(0, s8, s64) .widenScalarToNextPow2(0) // TODO: We could support sum-of-pow2's but the lowering code doesn't know @@ -229,12 +229,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { .clampMaxNumElements(0, s64, 1); getActionDefinitionsBuilder(G_STORE) - .legalForTypesWithMemSize({{s8, p0, 8}, - {s16, p0, 16}, - {s32, p0, 32}, - {s64, p0, 64}, - {p0, p0, 64}, - {v2s32, p0, 64}}) + .legalForTypesWithMemDesc({{s8, p0, 8, 8}, + {s16, p0, 16, 8}, + {s32, p0, 32, 8}, + {s64, p0, 64, 8}, + {p0, p0, 64, 8}, + {v2s32, p0, 64, 8}}) .clampScalar(0, s8, s64) .widenScalarToNextPow2(0) // TODO: We could support sum-of-pow2's but the lowering code doesn't know diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index c68190baf6f4a..4487bffacdc72 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -397,17 +397,18 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST, .clampScalar(0, S32, S64); + // FIXME: Handle alignment requirements. auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}) - .legalForTypesWithMemSize({ - {S32, GlobalPtr, 8}, - {S32, GlobalPtr, 16}, - {S32, LocalPtr, 8}, - {S32, LocalPtr, 16}, - {S32, PrivatePtr, 8}, - {S32, PrivatePtr, 16}}); + .legalForTypesWithMemDesc({ + {S32, GlobalPtr, 8, 8}, + {S32, GlobalPtr, 16, 8}, + {S32, LocalPtr, 8, 8}, + {S32, LocalPtr, 16, 8}, + {S32, PrivatePtr, 8, 8}, + {S32, PrivatePtr, 16, 8}}); if (ST.hasFlatAddressSpace()) { - ExtLoads.legalForTypesWithMemSize({{S32, FlatPtr, 8}, - {S32, FlatPtr, 16}}); + ExtLoads.legalForTypesWithMemDesc({{S32, FlatPtr, 8, 8}, + {S32, FlatPtr, 16, 8}}); } ExtLoads.clampScalar(0, S32, S32) diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp index 10320e5989718..2b217ce660ee2 100644 --- a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp +++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp @@ -131,12 +131,12 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) { // floating point to them. auto &LoadStoreBuilder = getActionDefinitionsBuilder({G_LOAD, G_STORE}) - .legalForTypesWithMemSize({ - {s1, p0, 8}, - {s8, p0, 8}, - {s16, p0, 16}, - {s32, p0, 32}, - {p0, p0, 32}}); + .legalForTypesWithMemDesc({ + {s1, p0, 8, 8}, + {s8, p0, 8, 8}, + {s16, p0, 16, 8}, + {s32, p0, 32, 8}, + {p0, p0, 32, 8}}); getActionDefinitionsBuilder(G_GEP).legalFor({{p0, s32}}); diff --git a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp index 7003f5395ee9b..4fddabcedfe10 100644 --- a/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp +++ b/llvm/lib/Target/Mips/MipsLegalizerInfo.cpp @@ -36,15 +36,15 @@ MipsLegalizerInfo::MipsLegalizerInfo(const MipsSubtarget &ST) { .lowerFor({{s32, s1}}); getActionDefinitionsBuilder({G_LOAD, G_STORE}) - .legalForTypesWithMemSize({{s32, p0, 8}, - {s32, p0, 16}, - {s32, p0, 32}, - {p0, p0, 32}}) + .legalForTypesWithMemDesc({{s32, p0, 8, 8}, + {s32, p0, 16, 8}, + {s32, p0, 32, 8}, + {p0, p0, 32, 8}}) .minScalar(0, s32); getActionDefinitionsBuilder({G_ZEXTLOAD, G_SEXTLOAD}) - .legalForTypesWithMemSize({{s32, p0, 8}, - {s32, p0, 16}}) + .legalForTypesWithMemDesc({{s32, p0, 8, 8}, + {s32, p0, 16, 8}}) .minScalar(0, s32); getActionDefinitionsBuilder(G_SELECT) diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp index 3617388c04e9d..b342143e13942 100644 --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp @@ -356,3 +356,52 @@ TEST(LegalizerInfoTest, RuleSets) { EXPECT_ACTION(Unsupported, 0, LLT(), LegalityQuery(G_AND, {v2s33})); } } + +TEST(LegalizerInfoTest, MMOAlignment) { + using namespace TargetOpcode; + + const LLT s32 = LLT::scalar(32); + const LLT p0 = LLT::pointer(0, 64); + + { + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_LOAD) + .legalForTypesWithMemDesc({{s32, p0, 32, 32}}); + + LI.computeTables(); + + EXPECT_ACTION(Legal, 0, LLT(), + LegalityQuery(G_LOAD, {s32, p0}, + LegalityQuery::MemDesc{ + 32, 32, AtomicOrdering::NotAtomic})); + EXPECT_ACTION(Unsupported, 0, LLT(), + LegalityQuery(G_LOAD, {s32, p0}, + LegalityQuery::MemDesc{ + 32, 16, AtomicOrdering::NotAtomic })); + EXPECT_ACTION(Unsupported, 0, LLT(), + LegalityQuery(G_LOAD, {s32, p0}, + LegalityQuery::MemDesc{ + 32, 8, AtomicOrdering::NotAtomic})); + } + + // Test that the maximum supported alignment value isn't truncated + { + // Maximum IR defined alignment in bytes. + const uint64_t MaxAlignment = UINT64_C(1) << 29; + const uint64_t MaxAlignInBits = 8 * MaxAlignment; + LegalizerInfo LI; + LI.getActionDefinitionsBuilder(G_LOAD) + .legalForTypesWithMemDesc({{s32, p0, 32, MaxAlignInBits}}); + + LI.computeTables(); + + EXPECT_ACTION(Legal, 0, LLT(), + LegalityQuery(G_LOAD, {s32, p0}, + LegalityQuery::MemDesc{32, + MaxAlignInBits, AtomicOrdering::NotAtomic})); + EXPECT_ACTION(Unsupported, 0, LLT(), + LegalityQuery(G_LOAD, {s32, p0}, + LegalityQuery::MemDesc{ + 32, 8, AtomicOrdering::NotAtomic })); + } +}