AMDGPU: Don't bitcast float typed atomic store in IR #90116
Merged
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
|
@llvm/pr-subscribers-llvm-transforms @llvm/pr-subscribers-llvm-selectiondag Author: Matt Arsenault (arsenm) ChangesImplement the promotion in the DAG. Depends #90113 Full diff: https://github.com/llvm/llvm-project/pull/90116.diff 5 Files Affected:
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index aa746f1c7b7b3b..f115a39a6953ce 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -530,6 +530,7 @@ namespace {
bool refineExtractVectorEltIntoMultipleNarrowExtractVectorElts(SDNode *N);
SDValue visitSTORE(SDNode *N);
+ SDValue visitATOMIC_STORE(SDNode *N);
SDValue visitLIFETIME_END(SDNode *N);
SDValue visitINSERT_VECTOR_ELT(SDNode *N);
SDValue visitEXTRACT_VECTOR_ELT(SDNode *N);
@@ -1909,6 +1910,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::BR_CC: return visitBR_CC(N);
case ISD::LOAD: return visitLOAD(N);
case ISD::STORE: return visitSTORE(N);
+ case ISD::ATOMIC_STORE: return visitATOMIC_STORE(N);
case ISD::INSERT_VECTOR_ELT: return visitINSERT_VECTOR_ELT(N);
case ISD::EXTRACT_VECTOR_ELT: return visitEXTRACT_VECTOR_ELT(N);
case ISD::BUILD_VECTOR: return visitBUILD_VECTOR(N);
@@ -21096,6 +21098,24 @@ SDValue DAGCombiner::replaceStoreOfInsertLoad(StoreSDNode *ST) {
ST->getMemOperand()->getFlags());
}
+SDValue DAGCombiner::visitATOMIC_STORE(SDNode *N) {
+ AtomicSDNode *ST = cast<AtomicSDNode>(N);
+ SDValue Val = ST->getVal();
+ EVT VT = Val.getValueType();
+ EVT MemVT = ST->getMemoryVT();
+
+ if (MemVT.bitsLT(VT)) { // Is truncating store
+ APInt TruncDemandedBits = APInt::getLowBitsSet(VT.getScalarSizeInBits(),
+ MemVT.getScalarSizeInBits());
+ // See if we can simplify the operation with SimplifyDemandedBits, which
+ // only works if the value has a single use.
+ if (SimplifyDemandedBits(Val, TruncDemandedBits))
+ return SDValue(N, 0);
+ }
+
+ return SDValue();
+}
+
SDValue DAGCombiner::visitSTORE(SDNode *N) {
StoreSDNode *ST = cast<StoreSDNode>(N);
SDValue Chain = ST->getChain();
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 24f69ea1b742a6..e2a22cbeb18c53 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4946,7 +4946,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
OVT = Node->getOperand(0).getSimpleValueType();
}
- if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
+ if (Node->getOpcode() == ISD::ATOMIC_STORE ||
+ Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
Node->getOpcode() == ISD::STRICT_FSETCC ||
Node->getOpcode() == ISD::STRICT_FSETCCS)
@@ -5557,7 +5558,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Results.push_back(CvtVec);
break;
}
- case ISD::ATOMIC_SWAP: {
+ case ISD::ATOMIC_SWAP:
+ case ISD::ATOMIC_STORE: {
AtomicSDNode *AM = cast<AtomicSDNode>(Node);
SDLoc SL(Node);
SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
@@ -5566,13 +5568,22 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
"unexpected atomic_swap with illegal type");
- SDValue NewAtomic
- = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
- DAG.getVTList(NVT, MVT::Other),
- { AM->getChain(), AM->getBasePtr(), CastVal },
- AM->getMemOperand());
- Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
- Results.push_back(NewAtomic.getValue(1));
+ SDValue Op0 = AM->getBasePtr();
+ SDValue Op1 = CastVal;
+
+ // ATOMIC_STORE uses a swapped operand order from every other AtomicSDNode,
+ // but really it should merge with ISD::STORE.
+ if (AM->getOpcode() == ISD::ATOMIC_STORE)
+ std::swap(Op0, Op1);
+
+ SDValue NewAtomic = DAG.getAtomic(AM->getOpcode(), SL, NVT, AM->getChain(),
+ Op0, Op1, AM->getMemOperand());
+
+ if (AM->getOpcode() != ISD::ATOMIC_STORE) {
+ Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
+ Results.push_back(NewAtomic.getValue(1));
+ } else
+ Results.push_back(NewAtomic);
break;
}
case ISD::SPLAT_VECTOR: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 7685bc73cf9652..5f21f65cab6273 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -2249,6 +2249,7 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
+ case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
}
// clang-format on
@@ -2371,6 +2372,23 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
ST->getMemOperand());
}
+SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
+ unsigned OpNo) {
+ AtomicSDNode *ST = cast<AtomicSDNode>(N);
+ SDValue Val = ST->getVal();
+ SDLoc DL(N);
+
+ SDValue Promoted = GetPromotedFloat(Val);
+ EVT VT = ST->getOperand(1).getValueType();
+ EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+
+ SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
+ DL, IVT, Promoted);
+
+ return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
+ ST->getBasePtr(), ST->getMemOperand());
+}
+
//===----------------------------------------------------------------------===//
// Float Result Promotion
//===----------------------------------------------------------------------===//
@@ -3154,6 +3172,9 @@ bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
+ case ISD::ATOMIC_STORE:
+ Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
+ break;
case ISD::STACKMAP:
Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
break;
@@ -3307,6 +3328,19 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
ST->getMemOperand());
}
+SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
+ unsigned OpNo) {
+ assert(OpNo == 1 && "Can only soften the stored value!");
+ AtomicSDNode *ST = cast<AtomicSDNode>(N);
+ SDValue Val = ST->getVal();
+ SDLoc dl(N);
+
+ SDValue Promoted = GetSoftPromotedHalf(Val);
+ return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
+ ST->getChain(), Promoted, ST->getBasePtr(),
+ ST->getMemOperand());
+}
+
SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 9c855e55855312..fad0067564bf4d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -708,6 +708,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
+ SDValue PromoteFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
@@ -751,6 +752,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfOp_SETCC(SDNode *N);
SDValue SoftPromoteHalfOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo);
+ SDValue SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo);
SDValue SoftPromoteHalfOp_PATCHPOINT(SDNode *N, unsigned OpNo);
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index f4a747784d1fd2..9e78b5e367d864 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -148,6 +148,19 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::LOAD, MVT::i128, Promote);
AddPromotedToType(ISD::LOAD, MVT::i128, MVT::v4i32);
+ setOperationAction(ISD::ATOMIC_STORE, MVT::f32, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::f32, MVT::i32);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::f64, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::f64, MVT::i64);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::f16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::f16, MVT::i16);
+
+ setOperationAction(ISD::ATOMIC_STORE, MVT::bf16, Promote);
+ AddPromotedToType(ISD::ATOMIC_STORE, MVT::bf16, MVT::i16);
+
+
// There are no 64-bit extloads. These should be done as a 32-bit extload and
// an extension to 64-bit.
for (MVT VT : MVT::integer_valuetypes())
|
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
arsenm
force-pushed
the
amdgpu-no-bitcast-fp-atomic-store
branch
3 times, most recently
from
abe66f7
to
1b66317
Compare
arsenm
force-pushed
the
amdgpu-no-bitcast-fp-atomic-store
branch
2 times, most recently
from
e88ccd5
to
b82cd2b
Compare
|
ping |
arsenm
force-pushed
the
amdgpu-no-bitcast-fp-atomic-store
branch
5 times, most recently
from
b646414
to
936d17e
Compare
|
ping |
Implement the promotion in the DAG.
arsenm
force-pushed
the
amdgpu-no-bitcast-fp-atomic-store
branch
from
936d17e
to
9625401
Compare
|
ping |
jyknight
approved these changes
May 7, 2024
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Implement the promotion in the DAG.
Depends #90113