Skip to content

Commit ad9d13d

Browse files
arsenmarsenm
committed
SelectionDAG: Swap operands of atomic_store
Irritatingly, atomic_store had operands in the opposite order from regular store. This made it difficult to share patterns between regular and atomic stores. There was a previous incomplete attempt to move atomic_store into the regular StoreSDNode which would be better. I think it was a mistake for all atomicrmw to swap the operand order, so maybe it's better to take this one step further. https://reviews.llvm.org/D123143
1 parent e0447ce commit ad9d13d

35 files changed

+266
-336
lines changed

‎llvm/include/llvm/CodeGen/SelectionDAGNodes.h

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1385,6 +1385,7 @@ class MemSDNode : public SDNode {
13851385
const SDValue &getBasePtr() const {
13861386
switch (getOpcode()) {
13871387
case ISD::STORE:
1388+
case ISD::ATOMIC_STORE:
13881389
case ISD::VP_STORE:
13891390
case ISD::MSTORE:
13901391
case ISD::VP_SCATTER:
@@ -1457,8 +1458,12 @@ class AtomicSDNode : public MemSDNode {
14571458
MMO->isAtomic()) && "then why are we using an AtomicSDNode?");
14581459
}
14591460

1460-
const SDValue &getBasePtr() const { return getOperand(1); }
1461-
const SDValue &getVal() const { return getOperand(2); }
1461+
const SDValue &getBasePtr() const {
1462+
return getOpcode() == ISD::ATOMIC_STORE ? getOperand(2) : getOperand(1);
1463+
}
1464+
const SDValue &getVal() const {
1465+
return getOpcode() == ISD::ATOMIC_STORE ? getOperand(1) : getOperand(2);
1466+
}
14621467

14631468
/// Returns true if this SDNode represents cmpxchg atomic operation, false
14641469
/// otherwise.

‎llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -203,18 +203,16 @@ def : GINodeEquiv<G_ICMP, setcc> {
203203
// separate nodes for them. This GINodeEquiv maps the non-atomic stores to
204204
// G_STORE with a non-atomic MachineMemOperand.
205205
def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = true; }
206-
207-
def : GINodeEquiv<G_LOAD, atomic_load> {
206+
def : GINodeEquiv<G_STORE, atomic_store> {
208207
let CheckMMOIsNonAtomic = false;
209208
let CheckMMOIsAtomic = true;
210-
let IfSignExtend = G_SEXTLOAD;
211-
let IfZeroExtend = G_ZEXTLOAD;
212209
}
213210

214-
// Operands are swapped for atomic_store vs. regular store
215-
def : GINodeEquiv<G_STORE, atomic_store> {
211+
def : GINodeEquiv<G_LOAD, atomic_load> {
216212
let CheckMMOIsNonAtomic = false;
217213
let CheckMMOIsAtomic = true;
214+
let IfSignExtend = G_SEXTLOAD;
215+
let IfZeroExtend = G_ZEXTLOAD;
218216
}
219217

220218
def : GINodeEquiv<G_ATOMIC_CMPXCHG, atomic_cmp_swap>;

‎llvm/include/llvm/Target/TargetSelectionDAG.td

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -315,7 +315,7 @@ def SDTFPAtomic2 : SDTypeProfile<1, 2, [
315315
]>;
316316

317317
def SDTAtomicStore : SDTypeProfile<0, 2, [
318-
SDTCisPtrTy<0>, SDTCisInt<1>
318+
SDTCisInt<0>, SDTCisPtrTy<1>
319319
]>;
320320
def SDTAtomicLoad : SDTypeProfile<1, 1, [
321321
SDTCisInt<0>, SDTCisPtrTy<1>
@@ -1678,7 +1678,6 @@ defm atomic_load_min : binary_atomic_op<atomic_load_min>;
16781678
defm atomic_load_max : binary_atomic_op<atomic_load_max>;
16791679
defm atomic_load_umin : binary_atomic_op<atomic_load_umin>;
16801680
defm atomic_load_umax : binary_atomic_op<atomic_load_umax>;
1681-
defm atomic_store : binary_atomic_op<atomic_store>;
16821681
defm atomic_cmp_swap : ternary_atomic_op<atomic_cmp_swap>;
16831682

16841683
/// Atomic load which zeroes the excess high bits.
@@ -1874,6 +1873,35 @@ def trunc_masked_scatter_i32 :
18741873
MSN->getMemoryVT().getScalarType() == MVT::i32;
18751874
}]>;
18761875

1876+
1877+
def atomic_store_8 :
1878+
PatFrag<(ops node:$val, node:$ptr),
1879+
(atomic_store node:$val, node:$ptr)> {
1880+
let IsAtomic = true;
1881+
let MemoryVT = i8;
1882+
}
1883+
1884+
def atomic_store_16 :
1885+
PatFrag<(ops node:$val, node:$ptr),
1886+
(atomic_store node:$val, node:$ptr)> {
1887+
let IsAtomic = true;
1888+
let MemoryVT = i16;
1889+
}
1890+
1891+
def atomic_store_32 :
1892+
PatFrag<(ops node:$val, node:$ptr),
1893+
(atomic_store node:$val, node:$ptr)> {
1894+
let IsAtomic = true;
1895+
let MemoryVT = i32;
1896+
}
1897+
1898+
def atomic_store_64 :
1899+
PatFrag<(ops node:$val, node:$ptr),
1900+
(atomic_store node:$val, node:$ptr)> {
1901+
let IsAtomic = true;
1902+
let MemoryVT = i64;
1903+
}
1904+
18771905
//===----------------------------------------------------------------------===//
18781906
// Selection DAG Pattern Support.
18791907
//

‎llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1043,7 +1043,7 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
10431043
}
10441044
case ISD::ATOMIC_STORE:
10451045
Action = TLI.getOperationAction(Node->getOpcode(),
1046-
Node->getOperand(2).getValueType());
1046+
Node->getOperand(1).getValueType());
10471047
break;
10481048
case ISD::SELECT_CC:
10491049
case ISD::STRICT_FSETCC:
@@ -3080,11 +3080,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
30803080
}
30813081
case ISD::ATOMIC_STORE: {
30823082
// There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3083-
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
3084-
cast<AtomicSDNode>(Node)->getMemoryVT(),
3085-
Node->getOperand(0),
3086-
Node->getOperand(1), Node->getOperand(2),
3087-
cast<AtomicSDNode>(Node)->getMemOperand());
3083+
SDValue Swap = DAG.getAtomic(
3084+
ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3085+
Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3086+
cast<AtomicSDNode>(Node)->getMemOperand());
30883087
Results.push_back(Swap.getValue(1));
30893088
break;
30903089
}

‎llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1956,9 +1956,9 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
19561956
}
19571957

19581958
SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1959-
SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1959+
SDValue Op1 = GetPromotedInteger(N->getOperand(1));
19601960
return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1961-
N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1961+
N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
19621962
}
19631963

19641964
SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
@@ -5505,11 +5505,10 @@ SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
55055505

55065506
SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
55075507
SDLoc dl(N);
5508-
SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
5509-
cast<AtomicSDNode>(N)->getMemoryVT(),
5510-
N->getOperand(0),
5511-
N->getOperand(1), N->getOperand(2),
5512-
cast<AtomicSDNode>(N)->getMemOperand());
5508+
SDValue Swap =
5509+
DAG.getAtomic(ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(N)->getMemoryVT(),
5510+
N->getOperand(0), N->getOperand(2), N->getOperand(1),
5511+
cast<AtomicSDNode>(N)->getMemOperand());
55135512
return Swap.getValue(1);
55145513
}
55155514

‎llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4873,8 +4873,8 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
48734873
DAG.setRoot(S);
48744874
return;
48754875
}
4876-
SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain,
4877-
Ptr, Val, MMO);
4876+
SDValue OutChain =
4877+
DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, Val, Ptr, MMO);
48784878

48794879
setValue(&I, OutChain);
48804880
DAG.setRoot(OutChain);

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5727,7 +5727,8 @@ SDValue AArch64TargetLowering::LowerStore128(SDValue Op,
57275727
StoreNode->getMergedOrdering() == AtomicOrdering::Unordered ||
57285728
StoreNode->getMergedOrdering() == AtomicOrdering::Monotonic);
57295729

5730-
SDValue Value = StoreNode->getOpcode() == ISD::STORE
5730+
SDValue Value = (StoreNode->getOpcode() == ISD::STORE ||
5731+
StoreNode->getOpcode() == ISD::ATOMIC_STORE)
57315732
? StoreNode->getOperand(1)
57325733
: StoreNode->getOperand(2);
57335734
SDLoc DL(Op);

‎llvm/lib/Target/AArch64/AArch64InstrAtomics.td

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -162,14 +162,14 @@ def : Pat<(f64 (bitconvert (i64 (relaxed_load<atomic_load_64>
162162

163163
// A store operation that actually needs release semantics.
164164
class releasing_store<PatFrag base>
165-
: PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
165+
: PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> {
166166
let IsAtomic = 1;
167167
let IsAtomicOrderingReleaseOrStronger = 1;
168168
}
169169

170170
// An atomic store operation that doesn't actually need to be atomic on AArch64.
171171
class relaxed_store<PatFrag base>
172-
: PatFrag<(ops node:$ptr, node:$val), (base node:$ptr, node:$val)> {
172+
: PatFrag<(ops node:$ptr, node:$val), (base node:$val, node:$ptr)> {
173173
let IsAtomic = 1;
174174
let IsAtomicOrderingReleaseOrStronger = 0;
175175
}

‎llvm/lib/Target/AMDGPU/AMDGPUInstructions.td

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -544,19 +544,18 @@ def truncstorei16_#as : PatFrag<(ops node:$val, node:$ptr),
544544
def store_hi16_#as : StoreHi16 <truncstorei16, i16>;
545545
def truncstorei8_hi16_#as : StoreHi16<truncstorei8, i8>;
546546
def truncstorei16_hi16_#as : StoreHi16<truncstorei16, i16>;
547-
548547
} // End let IsStore = 1, AddressSpaces = ...
549548

550549
let IsAtomic = 1, AddressSpaces = !cast<AddressSpaceList>("StoreAddress_"#as).AddrSpaces in {
551-
def atomic_store_8_#as : PatFrag<(ops node:$ptr, node:$val),
552-
(atomic_store_8 node:$ptr, node:$val)>;
553-
def atomic_store_16_#as : PatFrag<(ops node:$ptr, node:$val),
554-
(atomic_store_16 node:$ptr, node:$val)>;
555-
def atomic_store_32_#as : PatFrag<(ops node:$ptr, node:$val),
556-
(atomic_store_32 node:$ptr, node:$val)>;
557-
def atomic_store_64_#as : PatFrag<(ops node:$ptr, node:$val),
558-
(atomic_store_64 node:$ptr, node:$val)>;
559-
}
550+
def atomic_store_8_#as : PatFrag<(ops node:$val, node:$ptr),
551+
(atomic_store_8 node:$val, node:$ptr)>;
552+
def atomic_store_16_#as : PatFrag<(ops node:$val, node:$ptr),
553+
(atomic_store_16 node:$val, node:$ptr)>;
554+
def atomic_store_32_#as : PatFrag<(ops node:$val, node:$ptr),
555+
(atomic_store_32 node:$val, node:$ptr)>;
556+
def atomic_store_64_#as : PatFrag<(ops node:$val, node:$ptr),
557+
(atomic_store_64 node:$val, node:$ptr)>;
558+
} // End let IsAtomic = 1, AddressSpaces = ...
560559
} // End foreach as
561560

562561
multiclass noret_op {

‎llvm/lib/Target/AMDGPU/BUFInstructions.td

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1804,14 +1804,13 @@ defm : MUBUFScratchLoadPat_D16<BUFFER_LOAD_SBYTE_D16_OFFEN, BUFFER_LOAD_SBYTE_D1
18041804

18051805
multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo Instr_OFFSET,
18061806
ValueType vt, PatFrag atomic_st> {
1807-
// Store follows atomic op convention so address is first
18081807
def : GCNPat <
1809-
(atomic_st (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset), vt:$val),
1808+
(atomic_st vt:$val, (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset)),
18101809
(Instr_ADDR64 $val, $vaddr, $srsrc, $soffset, $offset)
18111810
>;
18121811

18131812
def : GCNPat <
1814-
(atomic_st (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset), vt:$val),
1813+
(atomic_st vt:$val, (MUBUFOffset v4i32:$rsrc, i32:$soffset, i32:$offset)),
18151814
(Instr_OFFSET $val, $rsrc, $soffset, (as_i16imm $offset))
18161815
>;
18171816
}

‎llvm/lib/Target/AMDGPU/DSInstructions.td

Lines changed: 6 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -806,23 +806,6 @@ multiclass DSWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
806806
}
807807
}
808808

809-
// Irritatingly, atomic_store reverses the order of operands from a
810-
// normal store.
811-
class DSAtomicWritePat <DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat <
812-
(frag (DS1Addr1Offset i32:$ptr, i32:$offset), vt:$value),
813-
(inst $ptr, getVregSrcForVT<vt>.ret:$value, offset:$offset, (i1 0))
814-
>;
815-
816-
multiclass DSAtomicWritePat_mc <DS_Pseudo inst, ValueType vt, string frag> {
817-
let OtherPredicates = [LDSRequiresM0Init] in {
818-
def : DSAtomicWritePat<inst, vt, !cast<PatFrag>(frag#"_m0")>;
819-
}
820-
821-
let OtherPredicates = [NotLDSRequiresM0Init] in {
822-
def : DSAtomicWritePat<!cast<DS_Pseudo>(!cast<string>(inst)#"_gfx9"), vt, !cast<PatFrag>(frag)>;
823-
}
824-
}
825-
826809
defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
827810
defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
828811
defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
@@ -832,12 +815,12 @@ foreach vt = Reg32Types.types in {
832815
defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
833816
}
834817

835-
defm : DSAtomicWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
836-
defm : DSAtomicWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
837-
defm : DSAtomicWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
838-
defm : DSAtomicWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
839-
defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
840-
defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
818+
defm : DSWritePat_mc <DS_WRITE_B8, i16, "atomic_store_8_local">;
819+
defm : DSWritePat_mc <DS_WRITE_B8, i32, "atomic_store_8_local">;
820+
defm : DSWritePat_mc <DS_WRITE_B16, i16, "atomic_store_16_local">;
821+
defm : DSWritePat_mc <DS_WRITE_B16, i32, "atomic_store_16_local">;
822+
defm : DSWritePat_mc <DS_WRITE_B32, i32, "atomic_store_32_local">;
823+
defm : DSWritePat_mc <DS_WRITE_B64, i64, "atomic_store_64_local">;
841824

842825
let OtherPredicates = [HasD16LoadStore] in {
843826
def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>;

‎llvm/lib/Target/AMDGPU/FLATInstructions.td

Lines changed: 12 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -996,12 +996,6 @@ class GlobalStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
996996
(inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset)
997997
>;
998998

999-
class GlobalAtomicStoreSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
1000-
ValueType vt> : GCNPat <
1001-
(node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), vt:$data),
1002-
(inst $voffset, getVregSrcForVT<vt>.ret:$data, $saddr, $offset)
1003-
>;
1004-
1005999
class GlobalAtomicSaddrPat <FLAT_Pseudo inst, SDPatternOperator node,
10061000
ValueType vt, ValueType data_vt = vt> : GCNPat <
10071001
(vt (node (GlobalSAddr (i64 SReg_64:$saddr), (i32 VGPR_32:$voffset), i32:$offset), data_vt:$data)),
@@ -1024,13 +1018,6 @@ class FlatStoreSignedPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt
10241018
(inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset)
10251019
>;
10261020

1027-
class FlatStoreAtomicPat <FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> : GCNPat <
1028-
// atomic store follows atomic binop convention so the address comes
1029-
// first.
1030-
(node (FlatOffset i64:$vaddr, i32:$offset), vt:$data),
1031-
(inst $vaddr, getVregSrcForVT<vt>.ret:$data, $offset)
1032-
>;
1033-
10341021
class FlatStoreSignedAtomicPat <FLAT_Pseudo inst, SDPatternOperator node,
10351022
ValueType vt, ValueType data_vt = vt> : GCNPat <
10361023
// atomic store follows atomic binop convention so the address comes
@@ -1174,12 +1161,12 @@ def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>;
11741161
def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt>;
11751162
}
11761163

1177-
def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
1178-
def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
1179-
def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
1180-
def : FlatStoreAtomicPat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
1181-
def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
1182-
def : FlatStoreAtomicPat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
1164+
def : FlatStorePat <FLAT_STORE_DWORD, atomic_store_32_flat, i32>;
1165+
def : FlatStorePat <FLAT_STORE_DWORDX2, atomic_store_64_flat, i64>;
1166+
def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i32>;
1167+
def : FlatStorePat <FLAT_STORE_BYTE, atomic_store_8_flat, i16>;
1168+
def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i32>;
1169+
def : FlatStorePat <FLAT_STORE_SHORT, atomic_store_16_flat, i16>;
11831170

11841171
foreach as = [ "flat", "global" ] in {
11851172
defm : FlatAtomicPat <"FLAT_ATOMIC_ADD", "atomic_load_add_"#as, i32>;
@@ -1269,17 +1256,6 @@ multiclass GlobalFLATStorePats<FLAT_Pseudo inst, SDPatternOperator node,
12691256
}
12701257
}
12711258

1272-
// Deal with swapped operands for atomic_store vs. regular store
1273-
multiclass GlobalFLATAtomicStorePats<FLAT_Pseudo inst, SDPatternOperator node, ValueType vt> {
1274-
def : FlatStoreSignedAtomicPat <inst, node, vt> {
1275-
let AddedComplexity = 10;
1276-
}
1277-
1278-
def : GlobalAtomicStoreSaddrPat<!cast<FLAT_Pseudo>(!cast<string>(inst)#"_SADDR"), node, vt> {
1279-
let AddedComplexity = 11;
1280-
}
1281-
}
1282-
12831259
multiclass GlobalFLATAtomicPatsNoRtnBase<string inst, string node, ValueType vt,
12841260
ValueType data_vt = vt> {
12851261
let AddedComplexity = 11 in
@@ -1444,12 +1420,12 @@ defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>
14441420
defm : GlobalFLATLoadPats_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>;
14451421
}
14461422

1447-
defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
1448-
defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
1449-
defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
1450-
defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
1451-
defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
1452-
defm : GlobalFLATAtomicStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
1423+
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i32>;
1424+
defm : GlobalFLATStorePats <GLOBAL_STORE_BYTE, atomic_store_8_global, i16>;
1425+
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i32>;
1426+
defm : GlobalFLATStorePats <GLOBAL_STORE_SHORT, atomic_store_16_global, i16>;
1427+
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORD, atomic_store_32_global, i32>;
1428+
defm : GlobalFLATStorePats <GLOBAL_STORE_DWORDX2, atomic_store_64_global, i64>;
14531429

14541430
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_ADD", "atomic_load_add_global", i32>;
14551431
defm : GlobalFLATAtomicPats <"GLOBAL_ATOMIC_SUB", "atomic_load_sub_global", i32>;

‎llvm/lib/Target/AMDGPU/SIInstrInfo.td

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -604,14 +604,14 @@ def atomic_store_64_glue : PatFrag <
604604
}
605605

606606
let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces in {
607-
def atomic_store_8_local_m0 : PatFrag<(ops node:$ptr, node:$val),
608-
(atomic_store_8_glue node:$ptr, node:$val)>;
609-
def atomic_store_16_local_m0 : PatFrag<(ops node:$ptr, node:$val),
610-
(atomic_store_16_glue node:$ptr, node:$val)>;
611-
def atomic_store_32_local_m0 : PatFrag<(ops node:$ptr, node:$val),
612-
(atomic_store_32_glue node:$ptr, node:$val)>;
613-
def atomic_store_64_local_m0 : PatFrag<(ops node:$ptr, node:$val),
614-
(atomic_store_64_glue node:$ptr, node:$val)>;
607+
def atomic_store_8_local_m0 : PatFrag<(ops node:$val, node:$ptr),
608+
(atomic_store_8_glue node:$val, node:$ptr)>;
609+
def atomic_store_16_local_m0 : PatFrag<(ops node:$val, node:$ptr),
610+
(atomic_store_16_glue node:$val, node:$ptr)>;
611+
def atomic_store_32_local_m0 : PatFrag<(ops node:$val, node:$ptr),
612+
(atomic_store_32_glue node:$val, node:$ptr)>;
613+
def atomic_store_64_local_m0 : PatFrag<(ops node:$val, node:$ptr),
614+
(atomic_store_64_glue node:$val, node:$ptr)>;
615615
} // End let IsAtomic = 1, AddressSpaces = StoreAddress_local.AddrSpaces
616616

617617

0 commit comments

Comments
 (0)