Skip to content

Commit

Permalink
[AArch64] LSE Atomics reorg - part 1
Browse files Browse the repository at this point in the history 
Add memory synchronization semantics to LSE Atomics.

The memory semantics feature will be added in a subsequent patch.

In this patch, several corrections were added to the existing LSE Atomics
implementation, based on the ARM Errata D11904 from 05/12/2017.

Patch by: steleman

Differential Revision: https://reviews.llvm.org/D35319

llvm-svn: 310167
  • Loading branch information
@joelkevinjones
joelkevinjones committed Aug 5, 2017
1 parent fbefee9 commit 60711ca
Show file tree
Hide file tree
Showing 6 changed files with 4,416 additions and 130 deletions.
103 changes: 79 additions & 24 deletions llvm/include/llvm/Target/TargetSelectionDAG.td
View file
Expand Up @@ -1015,44 +1015,98 @@ def setle : PatFrag<(ops node:$lhs, node:$rhs),
def setne : PatFrag<(ops node:$lhs, node:$rhs),
(setcc node:$lhs, node:$rhs, SETNE)>;

def atomic_cmp_swap_8 :
PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
(atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;
}]>;
def atomic_cmp_swap_16 :
PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
(atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;
}]>;
def atomic_cmp_swap_32 :
PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
(atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;
}]>;
def atomic_cmp_swap_64 :
PatFrag<(ops node:$ptr, node:$cmp, node:$swap),
(atomic_cmp_swap node:$ptr, node:$cmp, node:$swap), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;
}]>;
multiclass binary_atomic_op_ord<SDNode atomic_op> {
def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Monotonic;
}]>;
def #NAME#_acquire : PatFrag<(ops node:$ptr, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Acquire;
}]>;
def #NAME#_release : PatFrag<(ops node:$ptr, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Release;
}]>;
def #NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::AcquireRelease;
}]>;
def #NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::SequentiallyConsistent;
}]>;
}

multiclass ternary_atomic_op_ord<SDNode atomic_op> {
def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Monotonic;
}]>;
def #NAME#_acquire : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Acquire;
}]>;
def #NAME#_release : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::Release;
}]>;
def #NAME#_acq_rel : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::AcquireRelease;
}]>;
def #NAME#_seq_cst : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(!cast<SDNode>(#NAME) node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getOrdering() == AtomicOrdering::SequentiallyConsistent;
}]>;
}

multiclass binary_atomic_op<SDNode atomic_op> {
def _8 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val), [{
(atomic_op node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;
}]>;
def _16 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val), [{
(atomic_op node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;
}]>;
def _32 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val), [{
(atomic_op node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;
}]>;
def _64 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val), [{
(atomic_op node:$ptr, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;
}]>;

defm NAME#_8 : binary_atomic_op_ord<atomic_op>;
defm NAME#_16 : binary_atomic_op_ord<atomic_op>;
defm NAME#_32 : binary_atomic_op_ord<atomic_op>;
defm NAME#_64 : binary_atomic_op_ord<atomic_op>;
}

multiclass ternary_atomic_op<SDNode atomic_op> {
def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i8;
}]>;
def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i16;
}]>;
def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i32;
}]>;
def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val), [{
return cast<AtomicSDNode>(N)->getMemoryVT() == MVT::i64;
}]>;

defm NAME#_8 : ternary_atomic_op_ord<atomic_op>;
defm NAME#_16 : ternary_atomic_op_ord<atomic_op>;
defm NAME#_32 : ternary_atomic_op_ord<atomic_op>;
defm NAME#_64 : ternary_atomic_op_ord<atomic_op>;
}

defm atomic_load_add : binary_atomic_op<atomic_load_add>;
Expand All @@ -1067,6 +1121,7 @@ defm atomic_load_max : binary_atomic_op<atomic_load_max>;
defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;
defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;
defm atomic_store : binary_atomic_op<atomic_store>;
defm atomic_cmp_swap : ternary_atomic_op<atomic_cmp_swap>;

def atomic_load_8 :
PatFrag<(ops node:$ptr),
Expand Down
112 changes: 64 additions & 48 deletions llvm/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp
View file
Expand Up @@ -55,6 +55,8 @@ class AArch64DeadRegisterDefinitions : public MachineFunctionPass {
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}

bool shouldSkip(const MachineInstr &MI, const MachineFunction &MF) const;
};
char AArch64DeadRegisterDefinitions::ID = 0;
} // end anonymous namespace
Expand All @@ -69,6 +71,63 @@ static bool usesFrameIndex(const MachineInstr &MI) {
return false;
}

bool
AArch64DeadRegisterDefinitions::shouldSkip(const MachineInstr &MI,
const MachineFunction &MF) const {
if (!MF.getSubtarget<AArch64Subtarget>().hasLSE())
return false;

#define CASE_AARCH64_ATOMIC_(PREFIX) \
case AArch64::PREFIX##X: \
case AArch64::PREFIX##W: \
case AArch64::PREFIX##H: \
case AArch64::PREFIX##B

for (const MachineMemOperand *MMO : MI.memoperands()) {
if (MMO->isAtomic()) {
unsigned Opcode = MI.getOpcode();
switch (Opcode) {
default:
return false;
break;

CASE_AARCH64_ATOMIC_(LDADDA):
CASE_AARCH64_ATOMIC_(LDADDAL):

CASE_AARCH64_ATOMIC_(LDCLRA):
CASE_AARCH64_ATOMIC_(LDCLRAL):

CASE_AARCH64_ATOMIC_(LDEORA):
CASE_AARCH64_ATOMIC_(LDEORAL):

CASE_AARCH64_ATOMIC_(LDSETA):
CASE_AARCH64_ATOMIC_(LDSETAL):

CASE_AARCH64_ATOMIC_(LDSMAXA):
CASE_AARCH64_ATOMIC_(LDSMAXAL):

CASE_AARCH64_ATOMIC_(LDSMINA):
CASE_AARCH64_ATOMIC_(LDSMINAL):

CASE_AARCH64_ATOMIC_(LDUMAXA):
CASE_AARCH64_ATOMIC_(LDUMAXAL):

CASE_AARCH64_ATOMIC_(LDUMINA):
CASE_AARCH64_ATOMIC_(LDUMINAL):

CASE_AARCH64_ATOMIC_(SWPA):
CASE_AARCH64_ATOMIC_(SWPAL):
return true;
break;
}
}
}

#undef CASE_AARCH64_ATOMIC_

return false;
}

void AArch64DeadRegisterDefinitions::processMachineBasicBlock(
MachineBasicBlock &MBB) {
const MachineFunction &MF = *MBB.getParent();
Expand All @@ -86,55 +145,12 @@ void AArch64DeadRegisterDefinitions::processMachineBasicBlock(
DEBUG(dbgs() << " Ignoring, XZR or WZR already used by the instruction\n");
continue;
}
if (MF.getSubtarget<AArch64Subtarget>().hasLSE()) {
// XZ/WZ for LSE can only be used when acquire semantics are not used,
// LDOPAL WZ is an invalid opcode.
switch (MI.getOpcode()) {
case AArch64::CASALB:
case AArch64::CASALH:
case AArch64::CASALW:
case AArch64::CASALX:
case AArch64::SWPALB:
case AArch64::SWPALH:
case AArch64::SWPALW:
case AArch64::SWPALX:
case AArch64::LDADDALB:
case AArch64::LDADDALH:
case AArch64::LDADDALW:
case AArch64::LDADDALX:
case AArch64::LDCLRALB:
case AArch64::LDCLRALH:
case AArch64::LDCLRALW:
case AArch64::LDCLRALX:
case AArch64::LDEORALB:
case AArch64::LDEORALH:
case AArch64::LDEORALW:
case AArch64::LDEORALX:
case AArch64::LDSETALB:
case AArch64::LDSETALH:
case AArch64::LDSETALW:
case AArch64::LDSETALX:
case AArch64::LDSMINALB:
case AArch64::LDSMINALH:
case AArch64::LDSMINALW:
case AArch64::LDSMINALX:
case AArch64::LDSMAXALB:
case AArch64::LDSMAXALH:
case AArch64::LDSMAXALW:
case AArch64::LDSMAXALX:
case AArch64::LDUMINALB:
case AArch64::LDUMINALH:
case AArch64::LDUMINALW:
case AArch64::LDUMINALX:
case AArch64::LDUMAXALB:
case AArch64::LDUMAXALH:
case AArch64::LDUMAXALW:
case AArch64::LDUMAXALX:
continue;
default:
break;
}

if (shouldSkip(MI, MF)) {
DEBUG(dbgs() << " Ignoring, Atomic instruction with acquire semantics using WZR/XZR\n");
continue;
}

const MCInstrDesc &Desc = MI.getDesc();
for (int I = 0, E = Desc.getNumDefs(); I != E; ++I) {
MachineOperand &MO = MI.getOperand(I);
Expand Down
68 changes: 14 additions & 54 deletions llvm/lib/Target/AArch64/AArch64InstrAtomics.td
View file
Expand Up @@ -407,57 +407,17 @@ def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch),
Sched<[WriteAtomic]>;

// v8.1 Atomic instructions:
def : Pat<(atomic_load_add_8 GPR64:$Rn, GPR32:$Rs), (LDADDALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_add_16 GPR64:$Rn, GPR32:$Rs), (LDADDALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_add_32 GPR64:$Rn, GPR32:$Rs), (LDADDALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_add_64 GPR64:$Rn, GPR64:$Rs), (LDADDALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_or_8 GPR64:$Rn, GPR32:$Rs), (LDSETALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_or_16 GPR64:$Rn, GPR32:$Rs), (LDSETALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_or_32 GPR64:$Rn, GPR32:$Rs), (LDSETALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_or_64 GPR64:$Rn, GPR64:$Rs), (LDSETALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_xor_8 GPR64:$Rn, GPR32:$Rs), (LDEORALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_xor_16 GPR64:$Rn, GPR32:$Rs), (LDEORALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_xor_32 GPR64:$Rn, GPR32:$Rs), (LDEORALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_xor_64 GPR64:$Rn, GPR64:$Rs), (LDEORALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_max_8 GPR64:$Rn, GPR32:$Rs), (LDSMAXALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_max_16 GPR64:$Rn, GPR32:$Rs), (LDSMAXALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_max_32 GPR64:$Rn, GPR32:$Rs), (LDSMAXALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_max_64 GPR64:$Rn, GPR64:$Rs), (LDSMAXALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_umax_8 GPR64:$Rn, GPR32:$Rs), (LDUMAXALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_umax_16 GPR64:$Rn, GPR32:$Rs), (LDUMAXALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_umax_32 GPR64:$Rn, GPR32:$Rs), (LDUMAXALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_umax_64 GPR64:$Rn, GPR64:$Rs), (LDUMAXALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_min_8 GPR64:$Rn, GPR32:$Rs), (LDSMINALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_min_16 GPR64:$Rn, GPR32:$Rs), (LDSMINALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_min_32 GPR64:$Rn, GPR32:$Rs), (LDSMINALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_min_64 GPR64:$Rn, GPR64:$Rs), (LDSMINALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_umin_8 GPR64:$Rn, GPR32:$Rs), (LDUMINALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_umin_16 GPR64:$Rn, GPR32:$Rs), (LDUMINALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_umin_32 GPR64:$Rn, GPR32:$Rs), (LDUMINALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_load_umin_64 GPR64:$Rn, GPR64:$Rs), (LDUMINALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_cmp_swap_8 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALB GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
def : Pat<(atomic_cmp_swap_16 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALH GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
def : Pat<(atomic_cmp_swap_32 GPR64:$Rn, GPR32:$Rold, GPR32:$Rnew), (CASALW GPR32:$Rold, GPR32:$Rnew, GPR64sp:$Rn)>;
def : Pat<(atomic_cmp_swap_64 GPR64:$Rn, GPR64:$Rold, GPR64:$Rnew), (CASALX GPR64:$Rold, GPR64:$Rnew, GPR64sp:$Rn)>;

def : Pat<(atomic_swap_8 GPR64:$Rn, GPR32:$Rs), (SWPALB GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_swap_16 GPR64:$Rn, GPR32:$Rs), (SWPALH GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_swap_32 GPR64:$Rn, GPR32:$Rs), (SWPALW GPR32:$Rs, GPR64sp:$Rn)>;
def : Pat<(atomic_swap_64 GPR64:$Rn, GPR64:$Rs), (SWPALX GPR64:$Rs, GPR64sp:$Rn)>;

def : Pat<(atomic_load_sub_8 GPR64:$Rn, GPR32:$Rs), (LDADDALB (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
def : Pat<(atomic_load_sub_16 GPR64:$Rn, GPR32:$Rs), (LDADDALH (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
def : Pat<(atomic_load_sub_32 GPR64:$Rn, GPR32:$Rs), (LDADDALW (SUBWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
def : Pat<(atomic_load_sub_64 GPR64:$Rn, GPR64:$Rs), (LDADDALX (SUBXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>;

def : Pat<(atomic_load_and_8 GPR64:$Rn, GPR32:$Rs), (LDCLRALB (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
def : Pat<(atomic_load_and_16 GPR64:$Rn, GPR32:$Rs), (LDCLRALH (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
def : Pat<(atomic_load_and_32 GPR64:$Rn, GPR32:$Rs), (LDCLRALW (ORNWrr WZR, GPR32:$Rs), GPR64sp:$Rn)>;
def : Pat<(atomic_load_and_64 GPR64:$Rn, GPR64:$Rs), (LDCLRALX (ORNXrr XZR, GPR64:$Rs), GPR64sp:$Rn)>;
let Predicates = [HasLSE] in {
defm : LDOPregister_patterns<"LDADD", "atomic_load_add">;
defm : LDOPregister_patterns<"LDSET", "atomic_load_or">;
defm : LDOPregister_patterns<"LDEOR", "atomic_load_xor">;
defm : LDOPregister_patterns<"LDSMAX", "atomic_load_max">;
defm : LDOPregister_patterns<"LDSMIN", "atomic_load_min">;
defm : LDOPregister_patterns<"LDUMAX", "atomic_load_umax">;
defm : LDOPregister_patterns<"LDUMIN", "atomic_load_umin">;
defm : LDOPregister_patterns<"SWP", "atomic_swap">;
defm : LDOPregister_patterns_mod<"LDADD", "atomic_load_sub", "SUB">;
defm : LDOPregister_patterns_mod<"LDCLR", "atomic_load_and", "ORN">;
defm : CASregister_patterns<"CAS", "atomic_cmp_swap">;
}

0 comments on commit 60711ca

Please sign in to comment.