Skip to content

Commit

Permalink
Add out-of-line-atomics support to GlobalISel (#74588)
Browse files Browse the repository at this point in the history 
This patch implement the GlobalISel counterpart to
4d7df43.
  • Loading branch information
@RoboTux
RoboTux committed Jan 4, 2024
1 parent 5ed11e7 commit ce61b0e
Show file tree
Hide file tree
Showing 12 changed files with 5,594 additions and 2,945 deletions.
5 changes: 5 additions & 0 deletions llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h
View file
Expand Up @@ -224,6 +224,11 @@ struct TypePairAndMemDesc {
}
};

/// True iff P is false.
template <typename Predicate> Predicate predNot(Predicate P) {
return [=](const LegalityQuery &Query) { return !P(Query); };
}

/// True iff P0 and P1 are true.
template<typename Predicate>
Predicate all(Predicate P0, Predicate P1) {
Expand Down
6 changes: 6 additions & 0 deletions llvm/include/llvm/CodeGen/RuntimeLibcalls.h
View file
Expand Up @@ -82,6 +82,12 @@ namespace RTLIB {
/// UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT);

/// Return the outline atomics value for the given atomic ordering, access
/// size and set of libcalls for a given atomic, or UNKNOWN_LIBCALL if there
/// is none.
Libcall getOutlineAtomicHelper(const Libcall (&LC)[5][4],
AtomicOrdering Order, uint64_t MemSize);

/// Return the outline atomics value for the given opcode, atomic ordering
/// and type, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT);
Expand Down
140 changes: 140 additions & 0 deletions llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
View file
Expand Up @@ -25,6 +25,7 @@
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
Expand Down Expand Up @@ -796,6 +797,132 @@ llvm::createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
return LegalizerHelper::Legalized;
}

static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI) {
unsigned Opc = MI.getOpcode();
auto &AtomicMI = cast<GMemOperation>(MI);
auto &MMO = AtomicMI.getMMO();
auto Ordering = MMO.getMergedOrdering();
LLT MemType = MMO.getMemoryType();
uint64_t MemSize = MemType.getSizeInBytes();
if (MemType.isVector())
return RTLIB::UNKNOWN_LIBCALL;

#define LCALLS(A, B) \
{ A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
#define LCALL5(A) \
LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
switch (Opc) {
case TargetOpcode::G_ATOMIC_CMPXCHG:
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_CAS)};
return getOutlineAtomicHelper(LC, Ordering, MemSize);
}
case TargetOpcode::G_ATOMICRMW_XCHG: {
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_SWP)};
return getOutlineAtomicHelper(LC, Ordering, MemSize);
}
case TargetOpcode::G_ATOMICRMW_ADD:
case TargetOpcode::G_ATOMICRMW_SUB: {
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDADD)};
return getOutlineAtomicHelper(LC, Ordering, MemSize);
}
case TargetOpcode::G_ATOMICRMW_AND: {
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDCLR)};
return getOutlineAtomicHelper(LC, Ordering, MemSize);
}
case TargetOpcode::G_ATOMICRMW_OR: {
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDSET)};
return getOutlineAtomicHelper(LC, Ordering, MemSize);
}
case TargetOpcode::G_ATOMICRMW_XOR: {
const RTLIB::Libcall LC[5][4] = {LCALL5(RTLIB::OUTLINE_ATOMIC_LDEOR)};
return getOutlineAtomicHelper(LC, Ordering, MemSize);
}
default:
return RTLIB::UNKNOWN_LIBCALL;
}
#undef LCALLS
#undef LCALL5
}

static LegalizerHelper::LegalizeResult
createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI) {
auto &Ctx = MIRBuilder.getMF().getFunction().getContext();

Type *RetTy;
SmallVector<Register> RetRegs;
SmallVector<CallLowering::ArgInfo, 3> Args;
unsigned Opc = MI.getOpcode();
switch (Opc) {
case TargetOpcode::G_ATOMIC_CMPXCHG:
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
Register Success;
LLT SuccessLLT;
auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
MI.getFirst4RegLLTs();
RetRegs.push_back(Ret);
RetTy = IntegerType::get(Ctx, RetLLT.getSizeInBits());
if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
std::tie(Ret, RetLLT, Success, SuccessLLT, Mem, MemLLT, Cmp, CmpLLT, New,
NewLLT) = MI.getFirst5RegLLTs();
RetRegs.push_back(Success);
RetTy = StructType::get(
Ctx, {RetTy, IntegerType::get(Ctx, SuccessLLT.getSizeInBits())});
}
Args.push_back({Cmp, IntegerType::get(Ctx, CmpLLT.getSizeInBits()), 0});
Args.push_back({New, IntegerType::get(Ctx, NewLLT.getSizeInBits()), 0});
Args.push_back({Mem, PointerType::get(Ctx, MemLLT.getAddressSpace()), 0});
break;
}
case TargetOpcode::G_ATOMICRMW_XCHG:
case TargetOpcode::G_ATOMICRMW_ADD:
case TargetOpcode::G_ATOMICRMW_SUB:
case TargetOpcode::G_ATOMICRMW_AND:
case TargetOpcode::G_ATOMICRMW_OR:
case TargetOpcode::G_ATOMICRMW_XOR: {
auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] = MI.getFirst3RegLLTs();
RetRegs.push_back(Ret);
RetTy = IntegerType::get(Ctx, RetLLT.getSizeInBits());
if (Opc == TargetOpcode::G_ATOMICRMW_AND)
Val =
MIRBuilder.buildXor(ValLLT, MIRBuilder.buildConstant(ValLLT, -1), Val)
.getReg(0);
else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
Val =
MIRBuilder.buildSub(ValLLT, MIRBuilder.buildConstant(ValLLT, 0), Val)
.getReg(0);
Args.push_back({Val, IntegerType::get(Ctx, ValLLT.getSizeInBits()), 0});
Args.push_back({Mem, PointerType::get(Ctx, MemLLT.getAddressSpace()), 0});
break;
}
default:
llvm_unreachable("unsupported opcode");
}

auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
RTLIB::Libcall RTLibcall = getOutlineAtomicLibcall(MI);
const char *Name = TLI.getLibcallName(RTLibcall);

// Unsupported libcall on the target.
if (!Name) {
LLVM_DEBUG(dbgs() << ".. .. Could not find libcall name for "
<< MIRBuilder.getTII().getName(Opc) << "\n");
return LegalizerHelper::UnableToLegalize;
}

CallLowering::CallLoweringInfo Info;
Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
Info.Callee = MachineOperand::CreateES(Name);
Info.OrigRet = CallLowering::ArgInfo(RetRegs, RetTy, 0);

std::copy(Args.begin(), Args.end(), std::back_inserter(Info.OrigArgs));
if (!CLI.lowerCall(MIRBuilder, Info))
return LegalizerHelper::UnableToLegalize;

return LegalizerHelper::Legalized;
}

static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
Type *FromType) {
auto ToMVT = MVT::getVT(ToType);
Expand Down Expand Up @@ -1081,6 +1208,19 @@ LegalizerHelper::libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver) {
return Status;
break;
}
case TargetOpcode::G_ATOMICRMW_XCHG:
case TargetOpcode::G_ATOMICRMW_ADD:
case TargetOpcode::G_ATOMICRMW_SUB:
case TargetOpcode::G_ATOMICRMW_AND:
case TargetOpcode::G_ATOMICRMW_OR:
case TargetOpcode::G_ATOMICRMW_XOR:
case TargetOpcode::G_ATOMIC_CMPXCHG:
case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
auto Status = createAtomicLibcall(MIRBuilder, MI);
if (Status != Legalized)
return Status;
break;
}
case TargetOpcode::G_BZERO:
case TargetOpcode::G_MEMCPY:
case TargetOpcode::G_MEMMOVE:
Expand Down
41 changes: 26 additions & 15 deletions llvm/lib/CodeGen/TargetLoweringBase.cpp
View file
Expand Up @@ -520,27 +520,28 @@ RTLIB::Libcall RTLIB::getFREXP(EVT RetVT) {
FREXP_PPCF128);
}

RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
MVT VT) {
RTLIB::Libcall RTLIB::getOutlineAtomicHelper(const Libcall (&LC)[5][4],
AtomicOrdering Order,
uint64_t MemSize) {
unsigned ModeN, ModelN;
switch (VT.SimpleTy) {
case MVT::i8:
switch (MemSize) {
case 1:
ModeN = 0;
break;
case MVT::i16:
case 2:
ModeN = 1;
break;
case MVT::i32:
case 4:
ModeN = 2;
break;
case MVT::i64:
case 8:
ModeN = 3;
break;
case MVT::i128:
case 16:
ModeN = 4;
break;
default:
return UNKNOWN_LIBCALL;
return RTLIB::UNKNOWN_LIBCALL;
}

switch (Order) {
Expand All @@ -561,34 +562,44 @@ RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
return UNKNOWN_LIBCALL;
}

return LC[ModeN][ModelN];
}

RTLIB::Libcall RTLIB::getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order,
MVT VT) {
unsigned ModeN, ModelN;
if (!VT.isScalarInteger())
return UNKNOWN_LIBCALL;
uint64_t MemSize = VT.getScalarSizeInBits() / 8;

#define LCALLS(A, B) \
{ A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
#define LCALL5(A) \
LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
switch (Opc) {
case ISD::ATOMIC_CMP_SWAP: {
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_CAS)};
return LC[ModeN][ModelN];
return getOutlineAtomicHelper(LC, Order, MemSize);
}
case ISD::ATOMIC_SWAP: {
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_SWP)};
return LC[ModeN][ModelN];
return getOutlineAtomicHelper(LC, Order, MemSize);
}
case ISD::ATOMIC_LOAD_ADD: {
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDADD)};
return LC[ModeN][ModelN];
return getOutlineAtomicHelper(LC, Order, MemSize);
}
case ISD::ATOMIC_LOAD_OR: {
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDSET)};
return LC[ModeN][ModelN];
return getOutlineAtomicHelper(LC, Order, MemSize);
}
case ISD::ATOMIC_LOAD_CLR: {
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDCLR)};
return LC[ModeN][ModelN];
return getOutlineAtomicHelper(LC, Order, MemSize);
}
case ISD::ATOMIC_LOAD_XOR: {
const Libcall LC[5][4] = {LCALL5(OUTLINE_ATOMIC_LDEOR)};
return LC[ModeN][ModelN];
return getOutlineAtomicHelper(LC, Order, MemSize);
}
default:
return UNKNOWN_LIBCALL;
Expand Down
30 changes: 24 additions & 6 deletions llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
View file
Expand Up @@ -765,17 +765,35 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.lowerIf(
all(typeInSet(0, {s8, s16, s32, s64, s128}), typeIs(2, p0)));

LegalityPredicate UseOutlineAtomics = [&ST](const LegalityQuery &Query) {
return ST.outlineAtomics() && !ST.hasLSE();
};

getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)))
.customIf([](const LegalityQuery &Query) {
return Query.Types[0].getSizeInBits() == 128;
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0),
predNot(UseOutlineAtomics)))
.customIf(all(typeIs(0, s128), predNot(UseOutlineAtomics)))
.customIf([UseOutlineAtomics](const LegalityQuery &Query) {
return Query.Types[0].getSizeInBits() == 128 &&
!UseOutlineAtomics(Query);
})
.libcallIf(all(typeInSet(0, {s8, s16, s32, s64, s128}), typeIs(1, p0),
UseOutlineAtomics))
.clampScalar(0, s32, s64);

getActionDefinitionsBuilder({G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD,
G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
G_ATOMICRMW_XOR})
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0),
predNot(UseOutlineAtomics)))
.libcallIf(all(typeInSet(0, {s8, s16, s32, s64}), typeIs(1, p0),
UseOutlineAtomics))
.clampScalar(0, s32, s64);

// Do not outline these atomics operations, as per comment in
// AArch64ISelLowering.cpp's shouldExpandAtomicRMWInIR().
getActionDefinitionsBuilder(
{G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
{G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
.legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)))
.clampScalar(0, s32, s64);

Expand Down

0 comments on commit ce61b0e

Please sign in to comment.