[AMDGPU] Perform scalar FMUL with bf16 more efficiently - #213969
Merged
Conversation
|
@llvm/pr-subscribers-backend-amdgpu Author: LU-JOHN ChangesPerform scalar FMUL with bf16 more efficiently. Utilize v2bf16 patterns. Full diff: https://github.com/llvm/llvm-project/pull/213969.diff 5 Files Affected:
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e55356ab3ea24..189ec3fe1e3c1 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -247,6 +247,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
// Widen scalar fcanonicalize to a v2bf16 operation with an unused high
// lane.
setOperationAction(ISD::FCANONICALIZE, MVT::bf16, Custom);
+ // Widen scalar fmul to a v2bf16 operation with an unused high lane.
+ setOperationAction(ISD::FMUL, MVT::bf16, Custom);
}
setOperationAction(ISD::FP_ROUND, MVT::bf16, Expand);
@@ -7718,7 +7720,6 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SMAX:
case ISD::UMIN:
case ISD::UMAX:
- case ISD::FMUL:
case ISD::FMINNUM_IEEE:
case ISD::FMAXNUM_IEEE:
case ISD::UADDSAT:
@@ -7727,8 +7728,9 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::SSUBSAT:
return splitBinaryVectorOp(Op, DAG);
case ISD::FADD:
+ case ISD::FMUL:
if (Op.getValueType() == MVT::bf16)
- return lowerScalarBF16FAdd(Op, DAG);
+ return lowerScalarBF16BinaryOp(Op, DAG);
return splitBinaryVectorOp(Op, DAG);
case ISD::FCANONICALIZE:
if (Op.getValueType() == MVT::bf16)
@@ -8797,8 +8799,8 @@ SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
DAG.getTargetConstant(0, DL, MVT::i32));
}
-SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
- SelectionDAG &DAG) const {
+SDValue SITargetLowering::lowerScalarBF16BinaryOp(SDValue Op,
+ SelectionDAG &DAG) const {
assert(Subtarget->hasBF16PackedInsts());
SDLoc DL(Op);
@@ -8815,10 +8817,10 @@ SDValue SITargetLowering::lowerScalarBF16FAdd(SDValue Op,
SDValue LHS = WidenOperand(Op.getOperand(0));
SDValue RHS = WidenOperand(Op.getOperand(1));
- SDValue Add =
- DAG.getNode(ISD::FADD, DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
+ SDValue Result =
+ DAG.getNode(Op.getOpcode(), DL, MVT::v2bf16, LHS, RHS, Op->getFlags());
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Add,
+ return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::bf16, Result,
DAG.getConstant(0, DL, MVT::i32));
}
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index f358f59311c79..3e0e5da94471f 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -166,7 +166,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
/// Custom lowering for ISD::FP_ROUND for MVT::f16.
SDValue lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
SDValue splitFP_ROUNDVectorOp(SDValue Op, SelectionDAG &DAG) const;
- SDValue lowerScalarBF16FAdd(SDValue Op, SelectionDAG &DAG) const;
+ SDValue lowerScalarBF16BinaryOp(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerScalarBF16FCanonicalize(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINNUM_FMAXNUM(SDValue Op, SelectionDAG &DAG) const;
SDValue lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/test/CodeGen/AMDGPU/bf16.ll b/llvm/test/CodeGen/AMDGPU/bf16.ll
index deb8f6e415b70..89006f14961eb 100644
--- a/llvm/test/CodeGen/AMDGPU/bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/bf16.ll
@@ -14442,7 +14442,7 @@ define bfloat @v_fmul_bf16(bfloat %a, bfloat %b) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_fma_mixlo_bf16 v0, v0, v1, 0 op_sel_hi:[1,1,0]
+; GFX1250-NEXT: v_pk_mul_bf16 v0, v0, v1
; GFX1250-NEXT: s_set_pc_i64 s[30:31]
%op = fmul bfloat %a, %b
ret bfloat %op
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
index 8de3847f1df8c..097adde1dece0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.cos.bf16.ll
@@ -15,11 +15,9 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; FAKE16-NEXT: s_mov_b32 s3, 0x3e230000
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; FAKE16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; FAKE16-NEXT: v_cos_bf16_e32 v0, v0
; FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
@@ -31,11 +29,9 @@ define amdgpu_kernel void @cos_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; REAL16-NEXT: s_mov_b32 s3, 0x3e230000
; REAL16-NEXT: v_mov_b32_e32 v1, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_cos_bf16_e32 v0.l, v0.l
; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
index ef331c8b0162d..d3eda06c9ddd1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.bf16.ll
@@ -15,11 +15,9 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; FAKE16-NEXT: v_nop
; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; FAKE16-NEXT: s_mov_b32 s3, 0x3e230000
; FAKE16-NEXT: v_mov_b32_e32 v1, 0
; FAKE16-NEXT: s_wait_kmcnt 0x0
-; FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; FAKE16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; FAKE16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; FAKE16-NEXT: v_sin_bf16_e32 v0, v0
; FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
@@ -31,11 +29,9 @@ define amdgpu_kernel void @sin_bf16(ptr addrspace(1) %out, bfloat %src) #1 {
; REAL16-NEXT: v_nop
; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; REAL16-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
-; REAL16-NEXT: s_mov_b32 s3, 0x3e230000
; REAL16-NEXT: v_mov_b32_e32 v1, 0
; REAL16-NEXT: s_wait_kmcnt 0x0
-; REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 2, 2), 0 ; msbs: dst=0 src0=0 src1=0 src2=0
-; REAL16-NEXT: v_fma_mixlo_bf16 v0, s2, s3, 0 op_sel_hi:[1,0,0]
+; REAL16-NEXT: v_pk_mul_bf16 v0, 0x3e23, s2
; REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; REAL16-NEXT: v_sin_bf16_e32 v0.l, v0.l
; REAL16-NEXT: global_store_b16 v1, v0, s[0:1]
|
LU-JOHN
force-pushed
the
bf16_fmul
branch
2 times, most recently
from
August 4, 2026 23:08
7bfbc3f to
5781d6e
Compare
🐧 Linux x64 Test Results
✅ The build succeeded and all tests passed. |
Signed-off-by: John Lu <John.Lu@amd.com>
sommerlukas
approved these changes
Aug 5, 2026
sommerlukas
left a comment
sommerlukas
left a comment
Contributor
There was a problem hiding this comment.
Looks good, are we doing the same for GlobalISel or are you planning to add that?
Contributor
Author
I'm working on doing the same for FMUL and Canonicalize for GlobalISel. |
jinge90
pushed a commit
to jinge90/llvm-project
that referenced
this pull request
Aug 6, 2026
Perform scalar FMUL with bf16 more efficiently. Utilize v2bf16 patterns. Signed-off-by: John Lu <John.Lu@amd.com>
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
2 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Perform scalar FMUL with bf16 more efficiently. Utilize v2bf16 patterns.