diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index adf4e0139e03c..9564c191b0d15 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -2267,19 +2267,23 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op, return DAG.getMergeValues(Res, DL); } -// (frem x, y) -> (fma (fneg (ftrunc (fdiv x, y))), y, x) +// (frem x, y) -> (fcopysign (fma (fneg (ftrunc (fdiv x, y))), y, x), x) +// The fcopysign is only required to get the correct result -0.0 when x is -0.0 +// (and y is non-zero). With NSZ it can be dropped. SDValue AMDGPUTargetLowering::LowerFREM(SDValue Op, SelectionDAG &DAG) const { SDLoc SL(Op); EVT VT = Op.getValueType(); auto Flags = Op->getFlags(); SDValue X = Op.getOperand(0); SDValue Y = Op.getOperand(1); + bool NSZ = mayIgnoreSignedZero(Op); SDValue Div = DAG.getNode(ISD::FDIV, SL, VT, X, Y, Flags); SDValue Trunc = DAG.getNode(ISD::FTRUNC, SL, VT, Div, Flags); SDValue Neg = DAG.getNode(ISD::FNEG, SL, VT, Trunc, Flags); // TODO: For f32 use FMAD instead if !hasFastFMA32? - return DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); + SDValue FMA = DAG.getNode(ISD::FMA, SL, VT, Neg, Y, X, Flags); + return NSZ ? FMA : DAG.getNode(ISD::FCOPYSIGN, SL, VT, FMA, X); } SDValue AMDGPUTargetLowering::LowerFCEIL(SDValue Op, SelectionDAG &DAG) const { diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 3d70ed150df12..42d8721cd323e 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -2341,6 +2341,14 @@ bool AMDGPULegalizerInfo::legalizeFceil( return true; } +static bool allowNoSignedZeros(const MachineFunction &MF, unsigned Flags) { + return (Flags & MachineInstr::FmNsz) || + MF.getTarget().Options.NoSignedZerosFPMath; +} + +// Legalize frem(x, y) -> copysign(x - y * trunc(x / y), x) +// The copysign is only required to get the correct result -0.0 when x is -0.0 +// (and y is non-zero). With NSZ it can be dropped. bool AMDGPULegalizerInfo::legalizeFrem( MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { @@ -2349,11 +2357,15 @@ bool AMDGPULegalizerInfo::legalizeFrem( Register Src1Reg = MI.getOperand(2).getReg(); auto Flags = MI.getFlags(); LLT Ty = MRI.getType(DstReg); + bool NSZ = allowNoSignedZeros(B.getMF(), Flags); auto Div = B.buildFDiv(Ty, Src0Reg, Src1Reg, Flags); auto Trunc = B.buildIntrinsicTrunc(Ty, Div, Flags); auto Neg = B.buildFNeg(Ty, Trunc, Flags); - B.buildFMA(DstReg, Neg, Src1Reg, Src0Reg, Flags); + DstOp FMADst = NSZ ? DstOp(DstReg) : DstOp(Ty); + auto FMA = B.buildFMA(FMADst, Neg, Src1Reg, Src0Reg, Flags); + if (!NSZ) + B.buildFCopysign(DstReg, FMA, Src0Reg); MI.eraseFromParent(); return true; } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll index 0417b97a53c0f..da7fca95d5eef 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/frem.ll @@ -29,7 +29,10 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0 +; CI-NEXT: v_or_b32_e32 v0, v1, v0 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[4:7], 0 @@ -46,12 +49,15 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_cvt_f32_f16_e32 v0, s2 ; VI-NEXT: v_cvt_f32_f16_e32 v2, s0 ; VI-NEXT: v_mov_b32_e32 v1, s0 +; VI-NEXT: s_and_b32 s0, s2, 0xffff8000 ; VI-NEXT: v_rcp_f32_e32 v2, v2 ; VI-NEXT: v_mul_f32_e32 v0, v0, v2 ; VI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; VI-NEXT: v_div_fixup_f16 v0, v0, v1, s2 ; VI-NEXT: v_trunc_f16_e32 v0, v0 -; VI-NEXT: v_fma_f16 v2, -v0, v1, s2 +; VI-NEXT: v_fma_f16 v0, -v0, v1, s2 +; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0 +; VI-NEXT: v_or_b32_e32 v2, s0, v0 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: flat_store_short v[0:1], v2 @@ -126,7 +132,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: v_rcp_f32_e32 v2, v1 ; CI-NEXT: v_mul_f32_e32 v2, v0, v2 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0 +; CI-NEXT: v_or_b32_e32 v0, v1, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; CI-NEXT: s_endpgm @@ -143,7 +152,10 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_rcp_f16_e32 v0, s0 ; VI-NEXT: v_mul_f16_e32 v0, s2, v0 ; VI-NEXT: v_trunc_f16_e32 v0, v0 -; VI-NEXT: v_fma_f16 v2, -v0, s0, v1 +; VI-NEXT: v_fma_f16 v0, -v0, s0, v1 +; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0 +; VI-NEXT: s_and_b32 s0, s2, 0xffff8000 +; VI-NEXT: v_or_b32_e32 v2, s0, v0 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: flat_store_short v[0:1], v2 @@ -178,11 +190,14 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_fma_f32 v1, -v1, v4, v2 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4 +; CI-NEXT: s_and_b32 s0, s2, 0x80000000 ; CI-NEXT: s_mov_b32 s6, -1 ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_div_fixup_f32 v1, v1, v0, s2 ; CI-NEXT: v_trunc_f32_e32 v1, v1 ; CI-NEXT: v_fma_f32 v0, -v1, v0, s2 +; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 ; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -207,9 +222,12 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_fma_f32 v1, -v1, v4, v2 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4 +; VI-NEXT: s_and_b32 s0, s2, 0x80000000 ; VI-NEXT: v_div_fixup_f32 v1, v1, v0, s2 ; VI-NEXT: v_trunc_f32_e32 v1, v1 -; VI-NEXT: v_fma_f32 v2, -v1, v0, s2 +; VI-NEXT: v_fma_f32 v0, -v1, v0, s2 +; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; VI-NEXT: v_or_b32_e32 v2, s0, v0 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: flat_store_dword v[0:1], v2 @@ -282,6 +300,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: v_mul_f32_e32 v0, s2, v0 ; CI-NEXT: v_trunc_f32_e32 v0, v0 ; CI-NEXT: v_fma_f32 v0, -v0, s0, v1 +; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; CI-NEXT: s_and_b32 s0, s2, 0x80000000 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 ; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -297,7 +318,10 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_rcp_f32_e32 v0, s0 ; VI-NEXT: v_mul_f32_e32 v0, s2, v0 ; VI-NEXT: v_trunc_f32_e32 v0, v0 -; VI-NEXT: v_fma_f32 v2, -v0, s0, v1 +; VI-NEXT: v_fma_f32 v0, -v0, s0, v1 +; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; VI-NEXT: s_and_b32 s0, s2, 0x80000000 +; VI-NEXT: v_or_b32_e32 v2, s0, v0 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: flat_store_dword v[0:1], v2 @@ -325,6 +349,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_mov_b32_e32 v1, s1 ; CI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[0:1], v[0:1], s[2:3] ; CI-NEXT: v_div_scale_f64 v[8:9], vcc, s[2:3], v[0:1], s[2:3] +; CI-NEXT: s_mov_b32 s0, 0 +; CI-NEXT: s_brev_b32 s1, 1 +; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] ; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3] ; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0 ; CI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5] @@ -336,6 +363,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[0:1], s[2:3] ; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] ; CI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[2:3] +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 +; CI-NEXT: v_or_b32_e32 v1, s1, v1 ; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -351,6 +381,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[0:1], v[0:1], s[2:3] ; VI-NEXT: v_div_scale_f64 v[8:9], vcc, s[2:3], v[0:1], s[2:3] +; VI-NEXT: s_mov_b32 s0, 0 +; VI-NEXT: s_brev_b32 s1, 1 +; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] ; VI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3] ; VI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0 ; VI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5] @@ -364,6 +397,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[2:3] ; VI-NEXT: v_mov_b32_e32 v2, s4 ; VI-NEXT: v_mov_b32_e32 v3, s5 +; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 +; VI-NEXT: v_or_b32_e32 v1, s1, v1 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: s_endpgm %r0 = load double, ptr addrspace(1) %in1, align 8 @@ -453,6 +489,12 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5] ; CI-NEXT: v_trunc_f64_e32 v[0:1], v[0:1] ; CI-NEXT: v_fma_f64 v[0:1], -v[0:1], s[0:1], v[2:3] +; CI-NEXT: s_mov_b32 s0, 0 +; CI-NEXT: s_brev_b32 s1, 1 +; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 +; CI-NEXT: v_or_b32_e32 v1, s1, v1 ; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -476,8 +518,14 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_fma_f64 v[0:1], v[6:7], v[0:1], v[4:5] ; VI-NEXT: v_trunc_f64_e32 v[0:1], v[0:1] ; VI-NEXT: v_fma_f64 v[0:1], -v[0:1], s[0:1], v[2:3] +; VI-NEXT: s_mov_b32 s0, 0 +; VI-NEXT: s_brev_b32 s1, 1 +; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[0:1] ; VI-NEXT: v_mov_b32_e32 v2, s4 ; VI-NEXT: v_mov_b32_e32 v3, s5 +; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 +; VI-NEXT: v_or_b32_e32 v1, s1, v1 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: s_endpgm ptr addrspace(1) %in2) #1 { @@ -515,7 +563,10 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0 +; CI-NEXT: v_or_b32_e32 v0, v1, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, s3 ; CI-NEXT: v_cvt_f32_f16_e32 v2, s6 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 @@ -536,7 +587,10 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v1 ; CI-NEXT: v_trunc_f32_e32 v3, v3 -; CI-NEXT: v_fma_f32 v1, -v3, v2, v1 +; CI-NEXT: v_fma_f32 v2, -v3, v2, v1 +; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; CI-NEXT: v_and_b32_e32 v1, 0x80000000, v1 +; CI-NEXT: v_or_b32_e32 v1, v2, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 ; CI-NEXT: v_or_b32_e32 v0, v0, v1 @@ -559,6 +613,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_rcp_f32_e32 v2, v2 ; VI-NEXT: s_lshr_b32 s1, s2, 16 ; VI-NEXT: v_rcp_f32_e32 v3, v3 +; VI-NEXT: s_and_b32 s0, s2, 0xffff8000 ; VI-NEXT: v_mul_f32_e32 v0, v0, v2 ; VI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; VI-NEXT: v_mov_b32_e32 v2, s3 @@ -566,13 +621,19 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_trunc_f16_e32 v0, v0 ; VI-NEXT: v_fma_f16 v0, -v0, v1, s2 ; VI-NEXT: v_cvt_f32_f16_e32 v1, s1 +; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 +; VI-NEXT: s_and_b32 s0, s1, 0xffff8000 ; VI-NEXT: v_mul_f32_e32 v1, v1, v3 ; VI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; VI-NEXT: v_div_fixup_f16 v1, v1, v2, s1 ; VI-NEXT: v_trunc_f16_e32 v1, v1 ; VI-NEXT: v_fma_f16 v1, -v1, v2, s1 +; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v1 +; VI-NEXT: v_or_b32_e32 v1, s0, v1 +; VI-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; VI-NEXT: v_or_b32_e32 v2, v0, v1 +; VI-NEXT: v_or_b32_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: flat_store_dword v[0:1], v2 @@ -614,7 +675,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_and_b32_e32 v0, 0x80000000, v0 +; CI-NEXT: v_or_b32_e32 v0, v1, v0 ; CI-NEXT: v_cvt_f32_f16_e32 v1, s8 ; CI-NEXT: v_cvt_f32_f16_e32 v2, s10 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 @@ -633,7 +697,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6 ; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v1 ; CI-NEXT: v_trunc_f32_e32 v3, v3 -; CI-NEXT: v_fma_f32 v1, -v3, v2, v1 +; CI-NEXT: v_fma_f32 v2, -v3, v2, v1 +; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 +; CI-NEXT: v_and_b32_e32 v1, 0x80000000, v1 +; CI-NEXT: v_or_b32_e32 v1, v2, v1 ; CI-NEXT: v_cvt_f32_f16_e32 v2, s3 ; CI-NEXT: v_cvt_f32_f16_e32 v3, s1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 @@ -651,7 +718,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 ; CI-NEXT: v_div_fixup_f32 v4, v4, v3, v2 ; CI-NEXT: v_trunc_f32_e32 v4, v4 -; CI-NEXT: v_fma_f32 v2, -v4, v3, v2 +; CI-NEXT: v_fma_f32 v3, -v4, v3, v2 +; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; CI-NEXT: v_and_b32_e32 v2, 0x80000000, v2 +; CI-NEXT: v_or_b32_e32 v2, v3, v2 ; CI-NEXT: v_cvt_f32_f16_e32 v3, s9 ; CI-NEXT: v_cvt_f32_f16_e32 v4, s11 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 @@ -673,7 +743,10 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_div_fixup_f32 v5, v5, v4, v3 ; CI-NEXT: v_trunc_f32_e32 v5, v5 -; CI-NEXT: v_fma_f32 v3, -v5, v4, v3 +; CI-NEXT: v_fma_f32 v4, -v5, v4, v3 +; CI-NEXT: v_and_b32_e32 v4, 0x7fffffff, v4 +; CI-NEXT: v_and_b32_e32 v3, 0x80000000, v3 +; CI-NEXT: v_or_b32_e32 v3, v4, v3 ; CI-NEXT: v_cvt_f16_f32_e32 v3, v3 ; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v3 ; CI-NEXT: v_or_b32_e32 v1, v2, v1 @@ -708,30 +781,44 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_cvt_f32_f16_e32 v1, s6 ; VI-NEXT: v_cvt_f32_f16_e32 v5, s9 ; VI-NEXT: s_lshr_b32 s7, s3, 16 +; VI-NEXT: v_and_b32_e32 v0, 0x7fff, v0 ; VI-NEXT: v_mul_f32_e32 v1, v1, v3 ; VI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_rcp_f32_e32 v5, v5 +; VI-NEXT: s_and_b32 s0, s2, 0xffff8000 ; VI-NEXT: v_div_fixup_f16 v1, v1, v2, s6 ; VI-NEXT: v_trunc_f16_e32 v1, v1 ; VI-NEXT: v_fma_f16 v1, -v1, v2, s6 ; VI-NEXT: v_cvt_f32_f16_e32 v2, s3 -; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; VI-NEXT: v_or_b32_e32 v0, v0, v1 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 +; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v1 +; VI-NEXT: s_and_b32 s0, s6, 0xffff8000 ; VI-NEXT: v_mul_f32_e32 v2, v2, v4 ; VI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; VI-NEXT: v_mov_b32_e32 v4, s9 +; VI-NEXT: v_or_b32_e32 v1, s0, v1 +; VI-NEXT: s_and_b32 s0, s3, 0xffff8000 ; VI-NEXT: v_div_fixup_f16 v2, v2, v3, s3 ; VI-NEXT: v_trunc_f16_e32 v2, v2 ; VI-NEXT: v_fma_f16 v2, -v2, v3, s3 ; VI-NEXT: v_cvt_f32_f16_e32 v3, s7 +; VI-NEXT: v_and_b32_e32 v2, 0x7fff, v2 +; VI-NEXT: v_or_b32_e32 v2, s0, v2 +; VI-NEXT: s_and_b32 s0, s7, 0xffff8000 ; VI-NEXT: v_mul_f32_e32 v3, v3, v5 ; VI-NEXT: v_cvt_f16_f32_e32 v3, v3 +; VI-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-NEXT: v_div_fixup_f16 v3, v3, v4, s7 ; VI-NEXT: v_trunc_f16_e32 v3, v3 ; VI-NEXT: v_fma_f16 v3, -v3, v4, s7 -; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v3 -; VI-NEXT: v_or_b32_e32 v1, v2, v1 +; VI-NEXT: v_and_b32_e32 v3, 0x7fff, v3 +; VI-NEXT: v_or_b32_e32 v3, s0, v3 +; VI-NEXT: v_and_b32_e32 v1, 0xffff, v3 +; VI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; VI-NEXT: v_or_b32_sdwa v1, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-NEXT: v_mov_b32_e32 v2, s4 ; VI-NEXT: v_mov_b32_e32 v3, s5 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] @@ -766,10 +853,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_fma_f32 v1, -v1, v4, v2 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; CI-NEXT: v_div_fmas_f32 v1, v1, v3, v4 +; CI-NEXT: s_and_b32 s0, s2, 0x80000000 ; CI-NEXT: v_div_fixup_f32 v1, v1, v0, s2 ; CI-NEXT: v_trunc_f32_e32 v1, v1 ; CI-NEXT: v_fma_f32 v0, -v1, v0, s2 +; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 ; CI-NEXT: v_mov_b32_e32 v1, s1 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 ; CI-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, s3 ; CI-NEXT: v_div_scale_f32 v3, vcc, s3, v1, s3 ; CI-NEXT: v_rcp_f32_e32 v4, v2 @@ -782,11 +872,14 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_fma_f32 v2, -v2, v5, v3 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 +; CI-NEXT: s_and_b32 s0, s3, 0x80000000 ; CI-NEXT: s_mov_b32 s6, -1 ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, s3 ; CI-NEXT: v_trunc_f32_e32 v2, v2 ; CI-NEXT: v_fma_f32 v1, -v2, v1, s3 +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_or_b32_e32 v1, s0, v1 ; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -811,10 +904,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f32 v1, -v1, v4, v2 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; VI-NEXT: v_div_fmas_f32 v1, v1, v3, v4 +; VI-NEXT: s_and_b32 s0, s2, 0x80000000 ; VI-NEXT: v_div_fixup_f32 v1, v1, v0, s2 ; VI-NEXT: v_trunc_f32_e32 v1, v1 ; VI-NEXT: v_fma_f32 v0, -v1, v0, s2 +; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 ; VI-NEXT: v_mov_b32_e32 v1, s1 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 ; VI-NEXT: v_div_scale_f32 v2, s[0:1], v1, v1, s3 ; VI-NEXT: v_div_scale_f32 v3, vcc, s3, v1, s3 ; VI-NEXT: v_rcp_f32_e32 v4, v2 @@ -827,10 +923,13 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f32 v2, -v2, v5, v3 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; VI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 +; VI-NEXT: s_and_b32 s0, s3, 0x80000000 ; VI-NEXT: v_div_fixup_f32 v2, v2, v1, s3 ; VI-NEXT: v_trunc_f32_e32 v2, v2 ; VI-NEXT: v_fma_f32 v1, -v2, v1, s3 +; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 ; VI-NEXT: v_mov_b32_e32 v2, s4 +; VI-NEXT: v_or_b32_e32 v1, s0, v1 ; VI-NEXT: v_mov_b32_e32 v3, s5 ; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1] ; VI-NEXT: s_endpgm @@ -869,6 +968,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_fma_f32 v0, -v1, v0, s0 ; CI-NEXT: v_mov_b32_e32 v1, s9 ; CI-NEXT: v_div_scale_f32 v2, s[6:7], v1, v1, s1 +; CI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; CI-NEXT: s_and_b32 s0, s0, 0x80000000 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 ; CI-NEXT: v_div_scale_f32 v3, vcc, s1, v1, s1 ; CI-NEXT: v_rcp_f32_e32 v4, v2 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -880,10 +982,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_fma_f32 v2, -v2, v5, v3 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 +; CI-NEXT: s_and_b32 s0, s1, 0x80000000 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, s1 ; CI-NEXT: v_trunc_f32_e32 v2, v2 ; CI-NEXT: v_fma_f32 v1, -v2, v1, s1 +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 ; CI-NEXT: v_mov_b32_e32 v2, s10 +; CI-NEXT: v_or_b32_e32 v1, s0, v1 ; CI-NEXT: v_div_scale_f32 v3, s[0:1], v2, v2, s2 ; CI-NEXT: v_div_scale_f32 v4, vcc, s2, v2, s2 ; CI-NEXT: v_rcp_f32_e32 v5, v3 @@ -896,10 +1001,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_fma_f32 v3, -v3, v6, v4 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6 +; CI-NEXT: s_and_b32 s0, s2, 0x80000000 ; CI-NEXT: v_div_fixup_f32 v3, v3, v2, s2 ; CI-NEXT: v_trunc_f32_e32 v3, v3 ; CI-NEXT: v_fma_f32 v2, -v3, v2, s2 +; CI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 ; CI-NEXT: v_mov_b32_e32 v3, s11 +; CI-NEXT: v_or_b32_e32 v2, s0, v2 ; CI-NEXT: v_div_scale_f32 v4, s[0:1], v3, v3, s3 ; CI-NEXT: v_div_scale_f32 v5, vcc, s3, v3, s3 ; CI-NEXT: v_rcp_f32_e32 v6, v4 @@ -912,11 +1020,14 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_fma_f32 v4, -v4, v7, v5 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 +; CI-NEXT: s_and_b32 s0, s3, 0x80000000 ; CI-NEXT: s_mov_b32 s6, -1 ; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_div_fixup_f32 v4, v4, v3, s3 ; CI-NEXT: v_trunc_f32_e32 v4, v4 ; CI-NEXT: v_fma_f32 v3, -v4, v3, s3 +; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; CI-NEXT: v_or_b32_e32 v3, s0, v3 ; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -946,6 +1057,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f32 v0, -v1, v0, s0 ; VI-NEXT: v_mov_b32_e32 v1, s9 ; VI-NEXT: v_div_scale_f32 v2, s[6:7], v1, v1, s1 +; VI-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0 +; VI-NEXT: s_and_b32 s0, s0, 0x80000000 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 ; VI-NEXT: v_div_scale_f32 v3, vcc, s1, v1, s1 ; VI-NEXT: v_rcp_f32_e32 v4, v2 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -957,10 +1071,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f32 v2, -v2, v5, v3 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; VI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 +; VI-NEXT: s_and_b32 s0, s1, 0x80000000 ; VI-NEXT: v_div_fixup_f32 v2, v2, v1, s1 ; VI-NEXT: v_trunc_f32_e32 v2, v2 ; VI-NEXT: v_fma_f32 v1, -v2, v1, s1 +; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 ; VI-NEXT: v_mov_b32_e32 v2, s10 +; VI-NEXT: v_or_b32_e32 v1, s0, v1 ; VI-NEXT: v_div_scale_f32 v3, s[0:1], v2, v2, s2 ; VI-NEXT: v_div_scale_f32 v4, vcc, s2, v2, s2 ; VI-NEXT: v_rcp_f32_e32 v5, v3 @@ -973,10 +1090,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f32 v3, -v3, v6, v4 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; VI-NEXT: v_div_fmas_f32 v3, v3, v5, v6 +; VI-NEXT: s_and_b32 s0, s2, 0x80000000 ; VI-NEXT: v_div_fixup_f32 v3, v3, v2, s2 ; VI-NEXT: v_trunc_f32_e32 v3, v3 ; VI-NEXT: v_fma_f32 v2, -v3, v2, s2 +; VI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v2 ; VI-NEXT: v_mov_b32_e32 v3, s11 +; VI-NEXT: v_or_b32_e32 v2, s0, v2 ; VI-NEXT: v_div_scale_f32 v4, s[0:1], v3, v3, s3 ; VI-NEXT: v_div_scale_f32 v5, vcc, s3, v3, s3 ; VI-NEXT: v_rcp_f32_e32 v6, v4 @@ -989,10 +1109,13 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f32 v4, -v4, v7, v5 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0 ; VI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 +; VI-NEXT: s_and_b32 s0, s3, 0x80000000 ; VI-NEXT: v_div_fixup_f32 v4, v4, v3, s3 ; VI-NEXT: v_trunc_f32_e32 v4, v4 ; VI-NEXT: v_fma_f32 v3, -v4, v3, s3 +; VI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 ; VI-NEXT: v_mov_b32_e32 v4, s4 +; VI-NEXT: v_or_b32_e32 v3, s0, v3 ; VI-NEXT: v_mov_b32_e32 v5, s5 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; VI-NEXT: s_endpgm @@ -1017,8 +1140,8 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_mov_b32_e32 v1, s9 ; CI-NEXT: v_div_scale_f64 v[2:3], s[6:7], v[0:1], v[0:1], s[0:1] ; CI-NEXT: v_div_scale_f64 v[8:9], vcc, s[0:1], v[0:1], s[0:1] -; CI-NEXT: s_mov_b32 s6, -1 -; CI-NEXT: s_mov_b32 s7, 0xf000 +; CI-NEXT: s_mov_b32 s6, 0 +; CI-NEXT: s_brev_b32 s7, 1 ; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3] ; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0 ; CI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5] @@ -1031,9 +1154,16 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] ; CI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[0:1] ; CI-NEXT: v_mov_b32_e32 v2, s10 +; CI-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7] ; CI-NEXT: v_mov_b32_e32 v3, s11 -; CI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3] ; CI-NEXT: v_div_scale_f64 v[10:11], vcc, s[2:3], v[2:3], s[2:3] +; CI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; CI-NEXT: v_or_b32_e32 v0, s0, v0 +; CI-NEXT: v_or_b32_e32 v1, s1, v1 +; CI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3] +; CI-NEXT: s_and_b64 s[0:1], s[2:3], s[6:7] +; CI-NEXT: s_mov_b32 s6, -1 +; CI-NEXT: s_mov_b32 s7, 0xf000 ; CI-NEXT: v_rcp_f64_e32 v[6:7], v[4:5] ; CI-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0 ; CI-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7] @@ -1045,6 +1175,9 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], s[2:3] ; CI-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] ; CI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], s[2:3] +; CI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; CI-NEXT: v_or_b32_e32 v2, s0, v2 +; CI-NEXT: v_or_b32_e32 v3, s1, v3 ; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 ; CI-NEXT: s_endpgm ; @@ -1060,6 +1193,8 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_mov_b32_e32 v1, s9 ; VI-NEXT: v_div_scale_f64 v[2:3], s[6:7], v[0:1], v[0:1], s[0:1] ; VI-NEXT: v_div_scale_f64 v[8:9], vcc, s[0:1], v[0:1], s[0:1] +; VI-NEXT: s_mov_b32 s6, 0 +; VI-NEXT: s_brev_b32 s7, 1 ; VI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3] ; VI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0 ; VI-NEXT: v_fma_f64 v[4:5], v[4:5], v[6:7], v[4:5] @@ -1072,9 +1207,14 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] ; VI-NEXT: v_fma_f64 v[0:1], -v[2:3], v[0:1], s[0:1] ; VI-NEXT: v_mov_b32_e32 v2, s10 +; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[6:7] ; VI-NEXT: v_mov_b32_e32 v3, s11 -; VI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3] ; VI-NEXT: v_div_scale_f64 v[10:11], vcc, s[2:3], v[2:3], s[2:3] +; VI-NEXT: v_and_b32_e32 v1, 0x7fffffff, v1 +; VI-NEXT: v_or_b32_e32 v0, s0, v0 +; VI-NEXT: v_or_b32_e32 v1, s1, v1 +; VI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], s[2:3] +; VI-NEXT: s_and_b64 s[0:1], s[2:3], s[6:7] ; VI-NEXT: v_rcp_f64_e32 v[6:7], v[4:5] ; VI-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0 ; VI-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7] @@ -1088,6 +1228,9 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], s[2:3] ; VI-NEXT: v_mov_b32_e32 v4, s4 ; VI-NEXT: v_mov_b32_e32 v5, s5 +; VI-NEXT: v_and_b32_e32 v3, 0x7fffffff, v3 +; VI-NEXT: v_or_b32_e32 v2, s0, v2 +; VI-NEXT: v_or_b32_e32 v3, s1, v3 ; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3] ; VI-NEXT: s_endpgm %gep2 = getelementptr <2 x double>, ptr addrspace(1) %in2, i32 4 diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll index 7d4393b653a75..39a58dde8f1a2 100644 --- a/llvm/test/CodeGen/AMDGPU/frem.ll +++ b/llvm/test/CodeGen/AMDGPU/frem.ll @@ -43,7 +43,9 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; SI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 ; SI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; SI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; SI-NEXT: s_brev_b32 s0, -2 +; SI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[8:11], 0 @@ -72,6 +74,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_div_scale_f32 v3, s[0:1], v1, v1, v0 ; CI-NEXT: v_div_scale_f32 v2, vcc, v0, v1, v0 +; CI-NEXT: s_brev_b32 s0, -2 ; CI-NEXT: v_rcp_f32_e32 v4, v3 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 ; CI-NEXT: v_fma_f32 v5, -v3, v4, 1.0 @@ -84,7 +87,8 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[8:11], 0 @@ -103,6 +107,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: flat_load_ushort v2, v[2:3] +; VI-NEXT: s_movk_i32 s0, 0x7fff ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(1) @@ -115,6 +120,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_div_fixup_f16 v3, v3, v2, v4 ; VI-NEXT: v_trunc_f16_e32 v3, v3 ; VI-NEXT: v_fma_f16 v2, -v3, v2, v4 +; VI-NEXT: v_bfi_b32 v2, s0, v2, v4 ; VI-NEXT: flat_store_short v[0:1], v2 ; VI-NEXT: s_endpgm ; @@ -123,6 +129,7 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_movk_i32 s0, 0x7fff ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ushort v1, v0, s[6:7] ; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] offset:8 @@ -132,7 +139,8 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX9-NEXT: v_mad_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0] ; GFX9-NEXT: v_div_fixup_f16 v3, v3, v2, v1 ; GFX9-NEXT: v_trunc_f16_e32 v3, v3 -; GFX9-NEXT: v_fma_f16 v1, -v3, v2, v1 +; GFX9-NEXT: v_fma_f16 v2, -v3, v2, v1 +; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1 ; GFX9-NEXT: global_store_short v0, v1, s[4:5] ; GFX9-NEXT: s_endpgm ; @@ -152,7 +160,8 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX10-NEXT: v_fma_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0] ; GFX10-NEXT: v_div_fixup_f16 v3, v3, v2, v1 ; GFX10-NEXT: v_trunc_f16_e32 v3, v3 -; GFX10-NEXT: v_fma_f16 v1, -v3, v2, v1 +; GFX10-NEXT: v_fma_f16 v2, -v3, v2, v1 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX10-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-NEXT: s_endpgm ; @@ -175,7 +184,9 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX11-NEXT: v_div_fixup_f16 v3, v3, v2, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f16_e32 v3, v3 -; GFX11-NEXT: v_fma_f16 v1, -v3, v2, v1 +; GFX11-NEXT: v_fma_f16 v2, -v3, v2, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX11-NEXT: global_store_b16 v0, v1, s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -201,7 +212,9 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX1150-NEXT: v_trunc_f16_e32 v3, v3 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v3, 0x8000, v3 -; GFX1150-NEXT: v_fmac_f16_e32 v1, v3, v2 +; GFX1150-NEXT: v_fma_f16 v2, v3, v2, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX1150-NEXT: global_store_b16 v0, v1, s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -409,7 +422,9 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; SI-NEXT: v_rcp_f32_e32 v2, v1 ; SI-NEXT: v_mul_f32_e32 v2, v0, v2 ; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; SI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; SI-NEXT: s_brev_b32 s0, -2 +; SI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: buffer_store_short v0, off, s[8:11], 0 ; SI-NEXT: s_endpgm @@ -431,6 +446,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: s_mov_b32 s6, s10 ; CI-NEXT: s_mov_b32 s7, s11 ; CI-NEXT: buffer_load_ushort v0, off, s[4:7], 0 +; CI-NEXT: s_brev_b32 s0, -2 ; CI-NEXT: s_waitcnt vmcnt(1) ; CI-NEXT: v_cvt_f32_f16_e32 v1, v1 ; CI-NEXT: v_rcp_f32_e32 v2, v1 @@ -438,7 +454,8 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: v_cvt_f32_f16_e32 v0, v0 ; CI-NEXT: v_mul_f32_e32 v2, v0, v2 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: buffer_store_short v0, off, s[8:11], 0 ; CI-NEXT: s_endpgm @@ -456,6 +473,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: flat_load_ushort v2, v[2:3] +; VI-NEXT: s_movk_i32 s0, 0x7fff ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(0) @@ -463,6 +481,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_mul_f16_e32 v3, v4, v3 ; VI-NEXT: v_trunc_f16_e32 v3, v3 ; VI-NEXT: v_fma_f16 v2, -v3, v2, v4 +; VI-NEXT: v_bfi_b32 v2, s0, v2, v4 ; VI-NEXT: flat_store_short v[0:1], v2 ; VI-NEXT: s_endpgm ; @@ -471,6 +490,7 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_movk_i32 s0, 0x7fff ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_ushort v1, v0, s[6:7] ; GFX9-NEXT: global_load_ushort v2, v0, s[2:3] offset:8 @@ -478,7 +498,8 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; GFX9-NEXT: v_rcp_f16_e32 v3, v2 ; GFX9-NEXT: v_mul_f16_e32 v3, v1, v3 ; GFX9-NEXT: v_trunc_f16_e32 v3, v3 -; GFX9-NEXT: v_fma_f16 v1, -v3, v2, v1 +; GFX9-NEXT: v_fma_f16 v2, -v3, v2, v1 +; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1 ; GFX9-NEXT: global_store_short v0, v1, s[4:5] ; GFX9-NEXT: s_endpgm ; @@ -496,7 +517,8 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; GFX10-NEXT: v_rcp_f16_e32 v3, v2 ; GFX10-NEXT: v_mul_f16_e32 v3, v1, v3 ; GFX10-NEXT: v_trunc_f16_e32 v3, v3 -; GFX10-NEXT: v_fma_f16 v1, -v3, v2, v1 +; GFX10-NEXT: v_fma_f16 v2, -v3, v2, v1 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX10-NEXT: global_store_short v0, v1, s[4:5] ; GFX10-NEXT: s_endpgm ; @@ -516,7 +538,9 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; GFX11-NEXT: v_mul_f16_e32 v3, v1, v3 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f16_e32 v3, v3 -; GFX11-NEXT: v_fma_f16 v1, -v3, v2, v1 +; GFX11-NEXT: v_fma_f16 v2, -v3, v2, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX11-NEXT: global_store_b16 v0, v1, s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -539,7 +563,9 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace( ; GFX1150-NEXT: v_trunc_f16_e32 v3, v3 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v3, 0x8000, v3 -; GFX1150-NEXT: v_fmac_f16_e32 v1, v3, v2 +; GFX1150-NEXT: v_fma_f16 v2, v3, v2, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX1150-NEXT: global_store_b16 v0, v1, s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -586,7 +612,9 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; SI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 ; SI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; SI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; SI-NEXT: s_brev_b32 s0, -2 +; SI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0 ; SI-NEXT: s_endpgm ; @@ -610,6 +638,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_div_scale_f32 v3, s[0:1], v1, v1, v0 ; CI-NEXT: v_div_scale_f32 v2, vcc, v0, v1, v0 +; CI-NEXT: s_brev_b32 s0, -2 ; CI-NEXT: v_rcp_f32_e32 v4, v3 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 ; CI-NEXT: v_fma_f32 v5, -v3, v4, 1.0 @@ -622,7 +651,8 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_div_fmas_f32 v2, v2, v4, v5 ; CI-NEXT: v_div_fixup_f32 v2, v2, v1, v0 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; CI-NEXT: buffer_store_dword v0, off, s[8:11], 0 ; CI-NEXT: s_endpgm ; @@ -644,6 +674,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: v_div_scale_f32 v5, s[0:1], v2, v2, v4 ; VI-NEXT: v_div_scale_f32 v3, vcc, v4, v2, v4 +; VI-NEXT: s_brev_b32 s0, -2 ; VI-NEXT: v_rcp_f32_e32 v6, v5 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 ; VI-NEXT: v_fma_f32 v7, -v5, v6, 1.0 @@ -657,6 +688,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_div_fixup_f32 v3, v3, v2, v4 ; VI-NEXT: v_trunc_f32_e32 v3, v3 ; VI-NEXT: v_fma_f32 v2, -v3, v2, v4 +; VI-NEXT: v_bfi_b32 v2, s0, v2, v4 ; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: s_endpgm ; @@ -671,6 +703,7 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_div_scale_f32 v4, s[0:1], v2, v2, v1 ; GFX9-NEXT: v_div_scale_f32 v3, vcc, v1, v2, v1 +; GFX9-NEXT: s_brev_b32 s0, -2 ; GFX9-NEXT: v_rcp_f32_e32 v5, v4 ; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 ; GFX9-NEXT: v_fma_f32 v6, -v4, v5, 1.0 @@ -683,7 +716,8 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX9-NEXT: v_div_fmas_f32 v3, v3, v5, v6 ; GFX9-NEXT: v_div_fixup_f32 v3, v3, v2, v1 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_fma_f32 v1, -v3, v2, v1 +; GFX9-NEXT: v_fma_f32 v2, -v3, v2, v1 +; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1 ; GFX9-NEXT: global_store_dword v0, v1, s[4:5] ; GFX9-NEXT: s_endpgm ; @@ -712,7 +746,8 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX10-NEXT: v_div_fmas_f32 v3, v3, v5, v6 ; GFX10-NEXT: v_div_fixup_f32 v3, v3, v2, v1 ; GFX10-NEXT: v_trunc_f32_e32 v3, v3 -; GFX10-NEXT: v_fma_f32 v1, -v3, v2, v1 +; GFX10-NEXT: v_fma_f32 v2, -v3, v2, v1 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 ; GFX10-NEXT: global_store_dword v0, v1, s[4:5] ; GFX10-NEXT: s_endpgm ; @@ -747,7 +782,9 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX11-NEXT: v_div_fixup_f32 v3, v3, v2, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f32_e32 v3, v3 -; GFX11-NEXT: v_fma_f32 v1, -v3, v2, v1 +; GFX11-NEXT: v_fma_f32 v2, -v3, v2, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 ; GFX11-NEXT: global_store_b32 v0, v1, s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -785,7 +822,9 @@ define amdgpu_kernel void @frem_f32(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX1150-NEXT: v_trunc_f32_e32 v3, v3 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v3, 0x80000000, v3 -; GFX1150-NEXT: v_fmac_f32_e32 v1, v3, v2 +; GFX1150-NEXT: v_fma_f32 v2, v3, v2, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 ; GFX1150-NEXT: global_store_b32 v0, v1, s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -982,7 +1021,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; SI-NEXT: v_rcp_f32_e32 v2, v1 ; SI-NEXT: v_mul_f32_e32 v2, v0, v2 ; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; SI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; SI-NEXT: s_brev_b32 s0, -2 +; SI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0 ; SI-NEXT: s_endpgm ; @@ -1003,11 +1044,13 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: s_mov_b32 s3, s11 ; CI-NEXT: buffer_load_dword v0, off, s[4:7], 0 ; CI-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:16 +; CI-NEXT: s_brev_b32 s0, -2 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_rcp_f32_e32 v2, v1 ; CI-NEXT: v_mul_f32_e32 v2, v0, v2 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v0, -v2, v1, v0 +; CI-NEXT: v_fma_f32 v1, -v2, v1, v0 +; CI-NEXT: v_bfi_b32 v0, s0, v1, v0 ; CI-NEXT: buffer_store_dword v0, off, s[8:11], 0 ; CI-NEXT: s_endpgm ; @@ -1024,6 +1067,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: flat_load_dword v2, v[2:3] +; VI-NEXT: s_brev_b32 s0, -2 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(0) @@ -1031,6 +1075,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_mul_f32_e32 v3, v4, v3 ; VI-NEXT: v_trunc_f32_e32 v3, v3 ; VI-NEXT: v_fma_f32 v2, -v3, v2, v4 +; VI-NEXT: v_bfi_b32 v2, s0, v2, v4 ; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: s_endpgm ; @@ -1039,6 +1084,7 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_brev_b32 s0, -2 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dword v1, v0, s[6:7] ; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:16 @@ -1046,7 +1092,8 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; GFX9-NEXT: v_rcp_f32_e32 v3, v2 ; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_fma_f32 v1, -v3, v2, v1 +; GFX9-NEXT: v_fma_f32 v2, -v3, v2, v1 +; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1 ; GFX9-NEXT: global_store_dword v0, v1, s[4:5] ; GFX9-NEXT: s_endpgm ; @@ -1064,7 +1111,8 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; GFX10-NEXT: v_rcp_f32_e32 v3, v2 ; GFX10-NEXT: v_mul_f32_e32 v3, v1, v3 ; GFX10-NEXT: v_trunc_f32_e32 v3, v3 -; GFX10-NEXT: v_fma_f32 v1, -v3, v2, v1 +; GFX10-NEXT: v_fma_f32 v2, -v3, v2, v1 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 ; GFX10-NEXT: global_store_dword v0, v1, s[4:5] ; GFX10-NEXT: s_endpgm ; @@ -1084,7 +1132,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; GFX11-NEXT: v_mul_f32_e32 v3, v1, v3 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f32_e32 v3, v3 -; GFX11-NEXT: v_fma_f32 v1, -v3, v2, v1 +; GFX11-NEXT: v_fma_f32 v2, -v3, v2, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 ; GFX11-NEXT: global_store_b32 v0, v1, s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1107,7 +1157,9 @@ define amdgpu_kernel void @unsafe_frem_f32(ptr addrspace(1) %out, ptr addrspace( ; GFX1150-NEXT: v_trunc_f32_e32 v3, v3 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v3, 0x80000000, v3 -; GFX1150-NEXT: v_fmac_f32_e32 v1, v3, v2 +; GFX1150-NEXT: v_fma_f32 v2, v3, v2, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v2, v1 ; GFX1150-NEXT: global_store_b32 v0, v1, s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -1176,8 +1228,10 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; SI-NEXT: v_mov_b32_e32 v7, s2 ; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc ; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0 +; SI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; SI-NEXT: s_brev_b32 s0, -2 +; SI-NEXT: v_bfi_b32 v3, s0, v3, v1 +; SI-NEXT: buffer_store_dwordx2 v[2:3], off, s[4:7], 0 ; SI-NEXT: s_endpgm ; ; CI-LABEL: frem_f64: @@ -1199,6 +1253,7 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], v[0:1] +; CI-NEXT: s_brev_b32 s0, -2 ; CI-NEXT: v_rcp_f64_e32 v[6:7], v[4:5] ; CI-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0 ; CI-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7] @@ -1211,8 +1266,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; CI-NEXT: v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11] ; CI-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1] ; CI-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; CI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 +; CI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; CI-NEXT: v_bfi_b32 v3, s0, v3, v1 +; CI-NEXT: buffer_store_dwordx2 v[2:3], off, s[8:11], 0 ; CI-NEXT: s_endpgm ; ; VI-LABEL: frem_f64: @@ -1230,6 +1286,7 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(0) ; VI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[2:3] +; VI-NEXT: s_brev_b32 s0, -2 ; VI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] ; VI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 ; VI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] @@ -1242,8 +1299,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; VI-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13] ; VI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[2:3] ; VI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; VI-NEXT: v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3] -; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: v_fma_f64 v[4:5], -v[6:7], v[4:5], v[2:3] +; VI-NEXT: v_bfi_b32 v5, s0, v5, v3 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: frem_f64: @@ -1256,6 +1314,7 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX9-NEXT: global_load_dwordx2 v[2:3], v12, s[2:3] ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_div_scale_f64 v[4:5], s[0:1], v[2:3], v[2:3], v[0:1] +; GFX9-NEXT: s_brev_b32 s0, -2 ; GFX9-NEXT: v_rcp_f64_e32 v[6:7], v[4:5] ; GFX9-NEXT: v_fma_f64 v[8:9], -v[4:5], v[6:7], 1.0 ; GFX9-NEXT: v_fma_f64 v[6:7], v[6:7], v[8:9], v[6:7] @@ -1268,8 +1327,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX9-NEXT: v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11] ; GFX9-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1] ; GFX9-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX9-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX9-NEXT: global_store_dwordx2 v12, v[0:1], s[4:5] +; GFX9-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v1 +; GFX9-NEXT: global_store_dwordx2 v12, v[2:3], s[4:5] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: frem_f64: @@ -1295,8 +1355,9 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX10-NEXT: v_div_fmas_f64 v[4:5], v[4:5], v[6:7], v[10:11] ; GFX10-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1] ; GFX10-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX10-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX10-NEXT: global_store_dwordx2 v12, v[0:1], s[4:5] +; GFX10-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX10-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1 +; GFX10-NEXT: global_store_dwordx2 v12, v[2:3], s[4:5] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: frem_f64: @@ -1328,8 +1389,10 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX11-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX11-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX11-NEXT: global_store_b64 v12, v[0:1], s[4:5] +; GFX11-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1 +; GFX11-NEXT: global_store_b64 v12, v[2:3], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -1362,9 +1425,10 @@ define amdgpu_kernel void @frem_f64(ptr addrspace(1) %out, ptr addrspace(1) %in1 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_div_fixup_f64 v[4:5], v[4:5], v[2:3], v[0:1] ; GFX1150-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1150-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX1150-NEXT: global_store_b64 v12, v[0:1], s[4:5] +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX1150-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1 +; GFX1150-NEXT: global_store_b64 v12, v[2:3], s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1150-NEXT: s_endpgm @@ -1649,8 +1713,10 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; SI-NEXT: v_mov_b32_e32 v7, s6 ; SI-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc ; SI-NEXT: v_cndmask_b32_e32 v4, v6, v4, vcc -; SI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 +; SI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; SI-NEXT: s_brev_b32 s4, -2 +; SI-NEXT: v_bfi_b32 v3, s4, v3, v1 +; SI-NEXT: buffer_store_dwordx2 v[2:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; ; CI-LABEL: unsafe_frem_f64: @@ -1670,6 +1736,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: s_mov_b32 s3, s11 ; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 ; CI-NEXT: buffer_load_dwordx2 v[2:3], off, s[0:3], 0 +; CI-NEXT: s_brev_b32 s0, -2 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_rcp_f64_e32 v[4:5], v[2:3] ; CI-NEXT: v_fma_f64 v[6:7], -v[2:3], v[4:5], 1.0 @@ -1680,8 +1747,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; CI-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1] ; CI-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7] ; CI-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; CI-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 +; CI-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; CI-NEXT: v_bfi_b32 v3, s0, v3, v1 +; CI-NEXT: buffer_store_dwordx2 v[2:3], off, s[8:11], 0 ; CI-NEXT: s_endpgm ; ; VI-LABEL: unsafe_frem_f64: @@ -1695,6 +1763,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_mov_b32_e32 v5, s1 ; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] ; VI-NEXT: flat_load_dwordx2 v[4:5], v[4:5] +; VI-NEXT: s_brev_b32 s0, -2 ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(0) @@ -1707,8 +1776,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; VI-NEXT: v_fma_f64 v[10:11], -v[4:5], v[8:9], v[2:3] ; VI-NEXT: v_fma_f64 v[6:7], v[10:11], v[6:7], v[8:9] ; VI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; VI-NEXT: v_fma_f64 v[2:3], -v[6:7], v[4:5], v[2:3] -; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] +; VI-NEXT: v_fma_f64 v[4:5], -v[6:7], v[4:5], v[2:3] +; VI-NEXT: v_bfi_b32 v5, s0, v5, v3 +; VI-NEXT: flat_store_dwordx2 v[0:1], v[4:5] ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: unsafe_frem_f64: @@ -1716,6 +1786,7 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v10, 0 +; GFX9-NEXT: s_brev_b32 s0, -2 ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v10, s[6:7] ; GFX9-NEXT: global_load_dwordx2 v[2:3], v10, s[2:3] @@ -1729,8 +1800,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; GFX9-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1] ; GFX9-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7] ; GFX9-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX9-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX9-NEXT: global_store_dwordx2 v10, v[0:1], s[4:5] +; GFX9-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v1 +; GFX9-NEXT: global_store_dwordx2 v10, v[2:3], s[4:5] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: unsafe_frem_f64: @@ -1753,8 +1825,9 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; GFX10-NEXT: v_fma_f64 v[8:9], -v[2:3], v[6:7], v[0:1] ; GFX10-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7] ; GFX10-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX10-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX10-NEXT: global_store_dwordx2 v10, v[0:1], s[4:5] +; GFX10-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX10-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1 +; GFX10-NEXT: global_store_dwordx2 v10, v[2:3], s[4:5] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: unsafe_frem_f64: @@ -1782,8 +1855,10 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; GFX11-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX11-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX11-NEXT: global_store_b64 v10, v[0:1], s[4:5] +; GFX11-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1 +; GFX11-NEXT: global_store_b64 v10, v[2:3], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -1812,9 +1887,10 @@ define amdgpu_kernel void @unsafe_frem_f64(ptr addrspace(1) %out, ptr addrspace( ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f64 v[4:5], v[8:9], v[4:5], v[6:7] ; GFX1150-NEXT: v_trunc_f64_e32 v[4:5], v[4:5] -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) -; GFX1150-NEXT: v_fma_f64 v[0:1], -v[4:5], v[2:3], v[0:1] -; GFX1150-NEXT: global_store_b64 v10, v[0:1], s[4:5] +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_fma_f64 v[2:3], -v[4:5], v[2:3], v[0:1] +; GFX1150-NEXT: v_bfi_b32 v3, 0x7fffffff, v3, v1 +; GFX1150-NEXT: global_store_b64 v10, v[2:3], s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1150-NEXT: s_endpgm @@ -1866,7 +1942,9 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 ; SI-NEXT: v_div_fixup_f32 v4, v4, v2, v0 ; SI-NEXT: v_trunc_f32_e32 v4, v4 -; SI-NEXT: v_fma_f32 v0, -v4, v2, v0 +; SI-NEXT: v_fma_f32 v2, -v4, v2, v0 +; SI-NEXT: s_brev_b32 s6, -2 +; SI-NEXT: v_bfi_b32 v0, s6, v2, v0 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 @@ -1884,7 +1962,8 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v2, v2, v5, v6 ; SI-NEXT: v_div_fixup_f32 v2, v2, v3, v1 ; SI-NEXT: v_trunc_f32_e32 v2, v2 -; SI-NEXT: v_fma_f32 v1, -v2, v3, v1 +; SI-NEXT: v_fma_f32 v2, -v2, v3, v1 +; SI-NEXT: v_bfi_b32 v1, s6, v2, v1 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_or_b32_e32 v0, v1, v0 ; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0 @@ -1907,6 +1986,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s11, s3 ; CI-NEXT: buffer_load_dword v0, off, s[4:7], 0 ; CI-NEXT: buffer_load_dword v2, off, s[8:11], 0 offset:16 +; CI-NEXT: s_brev_b32 s6, -2 ; CI-NEXT: s_waitcnt vmcnt(1) ; CI-NEXT: v_cvt_f32_f16_e32 v1, v0 ; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0 @@ -1929,8 +2009,9 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 ; CI-NEXT: v_div_fixup_f32 v4, v4, v2, v0 ; CI-NEXT: v_trunc_f32_e32 v4, v4 -; CI-NEXT: v_fma_f32 v0, -v4, v2, v0 +; CI-NEXT: v_fma_f32 v2, -v4, v2, v0 ; CI-NEXT: v_div_scale_f32 v4, s[4:5], v3, v3, v1 +; CI-NEXT: v_bfi_b32 v0, s6, v2, v0 ; CI-NEXT: v_div_scale_f32 v2, vcc, v1, v3, v1 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 @@ -1947,7 +2028,8 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v2, v2, v5, v6 ; CI-NEXT: v_div_fixup_f32 v2, v2, v3, v1 ; CI-NEXT: v_trunc_f32_e32 v2, v2 -; CI-NEXT: v_fma_f32 v1, -v2, v3, v1 +; CI-NEXT: v_fma_f32 v2, -v2, v3, v1 +; CI-NEXT: v_bfi_b32 v1, s6, v2, v1 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; CI-NEXT: v_or_b32_e32 v0, v1, v0 ; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0 @@ -1966,30 +2048,33 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_mov_b32_e32 v3, s1 ; VI-NEXT: v_mov_b32_e32 v2, s0 ; VI-NEXT: flat_load_dword v2, v[2:3] +; VI-NEXT: s_movk_i32 s0, 0x7fff ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(1) -; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v4 -; VI-NEXT: v_cvt_f32_f16_e32 v5, v3 +; VI-NEXT: v_cvt_f32_f16_e32 v3, v4 ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v2 -; VI-NEXT: v_cvt_f32_f16_e32 v7, v6 -; VI-NEXT: v_rcp_f32_e32 v7, v7 -; VI-NEXT: v_mul_f32_e32 v5, v5, v7 -; VI-NEXT: v_cvt_f16_f32_e32 v5, v5 -; VI-NEXT: v_div_fixup_f16 v5, v5, v6, v3 -; VI-NEXT: v_trunc_f16_e32 v5, v5 -; VI-NEXT: v_fma_f16 v3, -v5, v6, v3 +; VI-NEXT: v_cvt_f32_f16_e32 v5, v2 +; VI-NEXT: v_rcp_f32_e32 v5, v5 +; VI-NEXT: v_mul_f32_e32 v3, v3, v5 +; VI-NEXT: v_cvt_f16_f32_e32 v3, v3 +; VI-NEXT: v_div_fixup_f16 v3, v3, v2, v4 +; VI-NEXT: v_trunc_f16_e32 v3, v3 +; VI-NEXT: v_fma_f16 v3, -v3, v2, v4 +; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; VI-NEXT: v_cvt_f32_f16_e32 v6, v2 +; VI-NEXT: v_bfi_b32 v3, s0, v3, v4 +; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v4 ; VI-NEXT: v_cvt_f32_f16_e32 v5, v4 -; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 ; VI-NEXT: v_rcp_f32_e32 v6, v6 ; VI-NEXT: v_mul_f32_e32 v5, v5, v6 ; VI-NEXT: v_cvt_f16_f32_e32 v5, v5 ; VI-NEXT: v_div_fixup_f16 v5, v5, v2, v4 ; VI-NEXT: v_trunc_f16_e32 v5, v5 ; VI-NEXT: v_fma_f16 v2, -v5, v2, v4 -; VI-NEXT: v_or_b32_e32 v2, v2, v3 +; VI-NEXT: v_bfi_b32 v2, s0, v2, v4 +; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; VI-NEXT: v_or_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-NEXT: flat_store_dword v[0:1], v2 ; VI-NEXT: s_endpgm ; @@ -1998,6 +2083,7 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-NEXT: s_movk_i32 s0, 0x7fff ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dword v1, v0, s[6:7] ; GFX9-NEXT: global_load_dword v2, v0, s[2:3] offset:16 @@ -2010,12 +2096,14 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_fma_f16 v3, -v3, v2, v1 ; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; GFX9-NEXT: v_cvt_f32_f16_e32 v4, v2 +; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v1 ; GFX9-NEXT: v_rcp_f32_e32 v4, v4 ; GFX9-NEXT: v_mad_mixlo_f16 v4, v1, v4, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX9-NEXT: v_div_fixup_f16 v4, v4, v2, v1 ; GFX9-NEXT: v_trunc_f16_e32 v4, v4 -; GFX9-NEXT: v_fma_f16 v1, -v4, v2, v1 +; GFX9-NEXT: v_fma_f16 v2, -v4, v2, v1 +; GFX9-NEXT: v_bfi_b32 v1, s0, v2, v1 ; GFX9-NEXT: v_pack_b32_f16 v1, v3, v1 ; GFX9-NEXT: global_store_dword v0, v1, s[4:5] ; GFX9-NEXT: s_endpgm @@ -2038,13 +2126,15 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_trunc_f16_e32 v3, v3 ; GFX10-NEXT: v_fma_f16 v3, -v3, v2, v1 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX10-NEXT: v_bfi_b32 v3, 0x7fff, v3, v1 ; GFX10-NEXT: v_cvt_f32_f16_e32 v4, v2 ; GFX10-NEXT: v_rcp_f32_e32 v4, v4 ; GFX10-NEXT: v_fma_mixlo_f16 v4, v1, v4, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_div_fixup_f16 v4, v4, v2, v1 ; GFX10-NEXT: v_trunc_f16_e32 v4, v4 -; GFX10-NEXT: v_fma_f16 v1, -v4, v2, v1 +; GFX10-NEXT: v_fma_f16 v2, -v4, v2, v1 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX10-NEXT: v_pack_b32_f16 v1, v3, v1 ; GFX10-NEXT: global_store_dword v0, v1, s[4:5] ; GFX10-NEXT: s_endpgm @@ -2070,17 +2160,20 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_trunc_f16_e32 v3, v3 ; GFX11-NEXT: v_fma_f16 v3, -v3, v2, v1 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_bfi_b32 v3, 0x7fff, v3, v1 ; GFX11-NEXT: v_cvt_f32_f16_e32 v4, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_rcp_f32_e32 v4, v4 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_mixlo_f16 v4, v1, v4, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_div_fixup_f16 v4, v4, v2, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f16_e32 v4, v4 +; GFX11-NEXT: v_fma_f16 v2, -v4, v2, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_f16 v1, -v4, v2, v1 +; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 ; GFX11-NEXT: v_pack_b32_f16 v1, v3, v1 ; GFX11-NEXT: global_store_b32 v0, v1, s[4:5] ; GFX11-NEXT: s_nop 0 @@ -2110,19 +2203,22 @@ define amdgpu_kernel void @frem_v2f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_trunc_f16_e32 v4, v4 ; GFX1150-NEXT: v_xor_b32_e32 v4, 0x8000, v4 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fmac_f16_e32 v5, v4, v3 -; GFX1150-NEXT: v_cvt_f32_f16_e32 v3, v2 -; GFX1150-NEXT: v_rcp_f32_e32 v3, v3 -; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fma_mixlo_f16 v3, v1, v3, 0 op_sel_hi:[1,0,0] -; GFX1150-NEXT: v_div_fixup_f16 v3, v3, v2, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_fma_f16 v3, v4, v3, v5 +; GFX1150-NEXT: v_cvt_f32_f16_e32 v4, v2 +; GFX1150-NEXT: v_bfi_b32 v3, 0x7fff, v3, v5 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: v_rcp_f32_e32 v4, v4 +; GFX1150-NEXT: v_fma_mixlo_f16 v4, v1, v4, 0 op_sel_hi:[1,0,0] ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_trunc_f16_e32 v3, v3 -; GFX1150-NEXT: v_xor_b32_e32 v3, 0x8000, v3 +; GFX1150-NEXT: v_div_fixup_f16 v4, v4, v2, v1 +; GFX1150-NEXT: v_trunc_f16_e32 v4, v4 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fmac_f16_e32 v1, v3, v2 -; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v5 +; GFX1150-NEXT: v_xor_b32_e32 v4, 0x8000, v4 +; GFX1150-NEXT: v_fma_f16 v2, v4, v2, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1 +; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v3 ; GFX1150-NEXT: global_store_b32 v0, v1, s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2183,6 +2279,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fixup_f32 v8, v8, v1, v5 ; SI-NEXT: v_trunc_f32_e32 v8, v8 ; SI-NEXT: v_fma_f32 v1, -v8, v1, v5 +; SI-NEXT: s_brev_b32 s6, -2 +; SI-NEXT: v_bfi_b32 v1, s6, v1, v5 ; SI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; SI-NEXT: v_cvt_f16_f32_e32 v1, v1 ; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 @@ -2200,7 +2298,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v5, v5, v9, v10 ; SI-NEXT: v_div_fixup_f32 v5, v5, v7, v4 ; SI-NEXT: v_trunc_f32_e32 v5, v5 -; SI-NEXT: v_fma_f32 v4, -v5, v7, v4 +; SI-NEXT: v_fma_f32 v5, -v5, v7, v4 +; SI-NEXT: v_bfi_b32 v4, s6, v5, v4 ; SI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; SI-NEXT: v_or_b32_e32 v1, v4, v1 ; SI-NEXT: v_div_scale_f32 v4, vcc, v3, v0, v3 @@ -2218,6 +2317,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fixup_f32 v4, v4, v0, v3 ; SI-NEXT: v_trunc_f32_e32 v4, v4 ; SI-NEXT: v_fma_f32 v0, -v4, v0, v3 +; SI-NEXT: v_bfi_b32 v0, s6, v0, v3 ; SI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 ; SI-NEXT: v_div_scale_f32 v3, vcc, v2, v6, v2 @@ -2234,7 +2334,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v3, v3, v5, v7 ; SI-NEXT: v_div_fixup_f32 v3, v3, v6, v2 ; SI-NEXT: v_trunc_f32_e32 v3, v3 -; SI-NEXT: v_fma_f32 v2, -v3, v6, v2 +; SI-NEXT: v_fma_f32 v3, -v3, v6, v2 +; SI-NEXT: v_bfi_b32 v2, s6, v3, v2 ; SI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; SI-NEXT: v_or_b32_e32 v0, v2, v0 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 @@ -2256,6 +2357,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s7, s3 ; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 ; CI-NEXT: s_mov_b32 s11, s3 +; CI-NEXT: s_brev_b32 s6, -2 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_cvt_f32_f16_e32 v2, v0 ; CI-NEXT: v_lshrrev_b32_e32 v0, 16, v0 @@ -2287,6 +2389,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_trunc_f32_e32 v8, v8 ; CI-NEXT: v_fma_f32 v1, -v8, v1, v5 ; CI-NEXT: v_div_scale_f32 v8, s[4:5], v7, v7, v4 +; CI-NEXT: v_bfi_b32 v1, s6, v1, v5 ; CI-NEXT: v_div_scale_f32 v5, vcc, v4, v7, v4 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 2, 2), 0 ; CI-NEXT: v_cvt_f16_f32_e32 v1, v1 @@ -2303,7 +2406,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v5, v5, v9, v10 ; CI-NEXT: v_div_fixup_f32 v5, v5, v7, v4 ; CI-NEXT: v_trunc_f32_e32 v5, v5 -; CI-NEXT: v_fma_f32 v4, -v5, v7, v4 +; CI-NEXT: v_fma_f32 v5, -v5, v7, v4 +; CI-NEXT: v_bfi_b32 v4, s6, v5, v4 ; CI-NEXT: v_div_scale_f32 v5, s[4:5], v0, v0, v3 ; CI-NEXT: v_cvt_f16_f32_e32 v4, v4 ; CI-NEXT: v_or_b32_e32 v1, v4, v1 @@ -2322,6 +2426,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_trunc_f32_e32 v4, v4 ; CI-NEXT: v_fma_f32 v0, -v4, v0, v3 ; CI-NEXT: v_div_scale_f32 v4, s[4:5], v6, v6, v2 +; CI-NEXT: v_bfi_b32 v0, s6, v0, v3 ; CI-NEXT: v_div_scale_f32 v3, vcc, v2, v6, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v0, v0 ; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 @@ -2337,7 +2442,8 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v7 ; CI-NEXT: v_div_fixup_f32 v3, v3, v6, v2 ; CI-NEXT: v_trunc_f32_e32 v3, v3 -; CI-NEXT: v_fma_f32 v2, -v3, v6, v2 +; CI-NEXT: v_fma_f32 v3, -v3, v6, v2 +; CI-NEXT: v_bfi_b32 v2, s6, v3, v2 ; CI-NEXT: v_cvt_f16_f32_e32 v2, v2 ; CI-NEXT: v_or_b32_e32 v0, v2, v0 ; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 @@ -2356,50 +2462,55 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: flat_load_dwordx2 v[4:5], v[4:5] ; VI-NEXT: v_mov_b32_e32 v3, s7 ; VI-NEXT: flat_load_dwordx2 v[2:3], v[2:3] +; VI-NEXT: s_movk_i32 s0, 0x7fff ; VI-NEXT: v_mov_b32_e32 v0, s4 ; VI-NEXT: v_mov_b32_e32 v1, s5 ; VI-NEXT: s_waitcnt vmcnt(1) -; VI-NEXT: v_lshrrev_b32_e32 v8, 16, v5 -; VI-NEXT: v_cvt_f32_f16_e32 v9, v8 +; VI-NEXT: v_cvt_f32_f16_e32 v7, v5 ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_lshrrev_b32_e32 v6, 16, v3 -; VI-NEXT: v_cvt_f32_f16_e32 v7, v6 -; VI-NEXT: v_rcp_f32_e32 v9, v9 -; VI-NEXT: v_mul_f32_e32 v7, v7, v9 -; VI-NEXT: v_cvt_f16_f32_e32 v7, v7 -; VI-NEXT: v_div_fixup_f16 v7, v7, v8, v6 -; VI-NEXT: v_trunc_f16_e32 v7, v7 -; VI-NEXT: v_fma_f16 v6, -v7, v8, v6 +; VI-NEXT: v_cvt_f32_f16_e32 v6, v3 +; VI-NEXT: v_rcp_f32_e32 v7, v7 +; VI-NEXT: v_mul_f32_e32 v6, v6, v7 +; VI-NEXT: v_cvt_f16_f32_e32 v6, v6 +; VI-NEXT: v_div_fixup_f16 v6, v6, v5, v3 +; VI-NEXT: v_trunc_f16_e32 v6, v6 +; VI-NEXT: v_fma_f16 v6, -v6, v5, v3 +; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v5 ; VI-NEXT: v_cvt_f32_f16_e32 v8, v5 +; VI-NEXT: v_bfi_b32 v6, s0, v6, v3 +; VI-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; VI-NEXT: v_cvt_f32_f16_e32 v7, v3 -; VI-NEXT: v_lshlrev_b32_e32 v6, 16, v6 ; VI-NEXT: v_rcp_f32_e32 v8, v8 ; VI-NEXT: v_mul_f32_e32 v7, v7, v8 ; VI-NEXT: v_cvt_f16_f32_e32 v7, v7 ; VI-NEXT: v_div_fixup_f16 v7, v7, v5, v3 ; VI-NEXT: v_trunc_f16_e32 v7, v7 -; VI-NEXT: v_fma_f16 v3, -v7, v5, v3 -; VI-NEXT: v_lshrrev_b32_e32 v7, 16, v4 -; VI-NEXT: v_cvt_f32_f16_e32 v8, v7 -; VI-NEXT: v_lshrrev_b32_e32 v5, 16, v2 -; VI-NEXT: v_or_b32_e32 v3, v3, v6 -; VI-NEXT: v_cvt_f32_f16_e32 v6, v5 -; VI-NEXT: v_rcp_f32_e32 v8, v8 -; VI-NEXT: v_mul_f32_e32 v6, v6, v8 -; VI-NEXT: v_cvt_f16_f32_e32 v6, v6 -; VI-NEXT: v_div_fixup_f16 v6, v6, v7, v5 -; VI-NEXT: v_trunc_f16_e32 v6, v6 -; VI-NEXT: v_fma_f16 v5, -v6, v7, v5 +; VI-NEXT: v_fma_f16 v5, -v7, v5, v3 +; VI-NEXT: v_bfi_b32 v3, s0, v5, v3 +; VI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; VI-NEXT: v_or_b32_sdwa v3, v6, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; VI-NEXT: v_cvt_f32_f16_e32 v6, v4 +; VI-NEXT: v_cvt_f32_f16_e32 v5, v2 +; VI-NEXT: v_rcp_f32_e32 v6, v6 +; VI-NEXT: v_mul_f32_e32 v5, v5, v6 +; VI-NEXT: v_cvt_f16_f32_e32 v5, v5 +; VI-NEXT: v_div_fixup_f16 v5, v5, v4, v2 +; VI-NEXT: v_trunc_f16_e32 v5, v5 +; VI-NEXT: v_fma_f16 v5, -v5, v4, v2 +; VI-NEXT: v_lshrrev_b32_e32 v4, 16, v4 ; VI-NEXT: v_cvt_f32_f16_e32 v7, v4 +; VI-NEXT: v_bfi_b32 v5, s0, v5, v2 +; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; VI-NEXT: v_cvt_f32_f16_e32 v6, v2 -; VI-NEXT: v_lshlrev_b32_e32 v5, 16, v5 ; VI-NEXT: v_rcp_f32_e32 v7, v7 ; VI-NEXT: v_mul_f32_e32 v6, v6, v7 ; VI-NEXT: v_cvt_f16_f32_e32 v6, v6 ; VI-NEXT: v_div_fixup_f16 v6, v6, v4, v2 ; VI-NEXT: v_trunc_f16_e32 v6, v6 -; VI-NEXT: v_fma_f16 v2, -v6, v4, v2 -; VI-NEXT: v_or_b32_e32 v2, v2, v5 +; VI-NEXT: v_fma_f16 v4, -v6, v4, v2 +; VI-NEXT: v_bfi_b32 v2, s0, v4, v2 +; VI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; VI-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD ; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] ; VI-NEXT: s_endpgm ; @@ -2408,6 +2519,7 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x34 ; GFX9-NEXT: v_mov_b32_e32 v4, 0 +; GFX9-NEXT: s_movk_i32 s0, 0x7fff ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7] ; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:32 @@ -2420,12 +2532,14 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_fma_f16 v5, -v5, v3, v1 ; GFX9-NEXT: v_lshrrev_b32_e32 v3, 16, v3 ; GFX9-NEXT: v_cvt_f32_f16_e32 v6, v3 +; GFX9-NEXT: v_bfi_b32 v5, s0, v5, v1 ; GFX9-NEXT: v_rcp_f32_e32 v6, v6 ; GFX9-NEXT: v_mad_mixlo_f16 v6, v1, v6, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX9-NEXT: v_div_fixup_f16 v6, v6, v3, v1 ; GFX9-NEXT: v_trunc_f16_e32 v6, v6 -; GFX9-NEXT: v_fma_f16 v1, -v6, v3, v1 +; GFX9-NEXT: v_fma_f16 v3, -v6, v3, v1 +; GFX9-NEXT: v_bfi_b32 v1, s0, v3, v1 ; GFX9-NEXT: v_cvt_f32_f16_e32 v3, v2 ; GFX9-NEXT: v_pack_b32_f16 v1, v5, v1 ; GFX9-NEXT: v_rcp_f32_e32 v3, v3 @@ -2435,12 +2549,14 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_fma_f16 v3, -v3, v2, v0 ; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2 ; GFX9-NEXT: v_cvt_f32_f16_e32 v5, v2 +; GFX9-NEXT: v_bfi_b32 v3, s0, v3, v0 ; GFX9-NEXT: v_rcp_f32_e32 v5, v5 ; GFX9-NEXT: v_mad_mixlo_f16 v5, v0, v5, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX9-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX9-NEXT: v_div_fixup_f16 v5, v5, v2, v0 ; GFX9-NEXT: v_trunc_f16_e32 v5, v5 -; GFX9-NEXT: v_fma_f16 v0, -v5, v2, v0 +; GFX9-NEXT: v_fma_f16 v2, -v5, v2, v0 +; GFX9-NEXT: v_bfi_b32 v0, s0, v2, v0 ; GFX9-NEXT: v_pack_b32_f16 v0, v3, v0 ; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5] ; GFX9-NEXT: s_endpgm @@ -2463,13 +2579,15 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_trunc_f16_e32 v5, v5 ; GFX10-NEXT: v_fma_f16 v5, -v5, v3, v1 ; GFX10-NEXT: v_lshrrev_b32_e32 v3, 16, v3 +; GFX10-NEXT: v_bfi_b32 v5, 0x7fff, v5, v1 ; GFX10-NEXT: v_cvt_f32_f16_e32 v6, v3 ; GFX10-NEXT: v_rcp_f32_e32 v6, v6 ; GFX10-NEXT: v_fma_mixlo_f16 v6, v1, v6, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX10-NEXT: v_lshrrev_b32_e32 v1, 16, v1 ; GFX10-NEXT: v_div_fixup_f16 v6, v6, v3, v1 ; GFX10-NEXT: v_trunc_f16_e32 v6, v6 -; GFX10-NEXT: v_fma_f16 v1, -v6, v3, v1 +; GFX10-NEXT: v_fma_f16 v3, -v6, v3, v1 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fff, v3, v1 ; GFX10-NEXT: v_cvt_f32_f16_e32 v3, v2 ; GFX10-NEXT: v_pack_b32_f16 v1, v5, v1 ; GFX10-NEXT: v_rcp_f32_e32 v3, v3 @@ -2478,13 +2596,15 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_trunc_f16_e32 v3, v3 ; GFX10-NEXT: v_fma_f16 v3, -v3, v2, v0 ; GFX10-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX10-NEXT: v_bfi_b32 v3, 0x7fff, v3, v0 ; GFX10-NEXT: v_cvt_f32_f16_e32 v5, v2 ; GFX10-NEXT: v_rcp_f32_e32 v5, v5 ; GFX10-NEXT: v_fma_mixlo_f16 v5, v0, v5, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX10-NEXT: v_lshrrev_b32_e32 v0, 16, v0 ; GFX10-NEXT: v_div_fixup_f16 v5, v5, v2, v0 ; GFX10-NEXT: v_trunc_f16_e32 v5, v5 -; GFX10-NEXT: v_fma_f16 v0, -v5, v2, v0 +; GFX10-NEXT: v_fma_f16 v2, -v5, v2, v0 +; GFX10-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0 ; GFX10-NEXT: v_pack_b32_f16 v0, v3, v0 ; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5] ; GFX10-NEXT: s_endpgm @@ -2510,17 +2630,20 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_trunc_f16_e32 v5, v5 ; GFX11-NEXT: v_fma_f16 v5, -v5, v3, v1 ; GFX11-NEXT: v_lshrrev_b32_e32 v3, 16, v3 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_bfi_b32 v5, 0x7fff, v5, v1 ; GFX11-NEXT: v_cvt_f32_f16_e32 v6, v3 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_rcp_f32_e32 v6, v6 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_mixlo_f16 v6, v1, v6, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX11-NEXT: v_lshrrev_b32_e32 v1, 16, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_div_fixup_f16 v6, v6, v3, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f16_e32 v6, v6 +; GFX11-NEXT: v_fma_f16 v3, -v6, v3, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX11-NEXT: v_fma_f16 v1, -v6, v3, v1 +; GFX11-NEXT: v_bfi_b32 v1, 0x7fff, v3, v1 ; GFX11-NEXT: v_cvt_f32_f16_e32 v3, v2 ; GFX11-NEXT: v_pack_b32_f16 v1, v5, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1) @@ -2532,17 +2655,20 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_trunc_f16_e32 v3, v3 ; GFX11-NEXT: v_fma_f16 v3, -v3, v2, v0 ; GFX11-NEXT: v_lshrrev_b32_e32 v2, 16, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_bfi_b32 v3, 0x7fff, v3, v0 ; GFX11-NEXT: v_cvt_f32_f16_e32 v5, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_rcp_f32_e32 v5, v5 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_mixlo_f16 v5, v0, v5, 0 op_sel:[1,0,0] op_sel_hi:[1,0,0] ; GFX11-NEXT: v_lshrrev_b32_e32 v0, 16, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_div_fixup_f16 v5, v5, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f16_e32 v5, v5 +; GFX11-NEXT: v_fma_f16 v2, -v5, v2, v0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_f16 v0, -v5, v2, v0 +; GFX11-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0 ; GFX11-NEXT: v_pack_b32_f16 v0, v3, v0 ; GFX11-NEXT: global_store_b64 v4, v[0:1], s[4:5] ; GFX11-NEXT: s_nop 0 @@ -2572,21 +2698,24 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_trunc_f16_e32 v6, v6 ; GFX1150-NEXT: v_xor_b32_e32 v6, 0x8000, v6 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2) -; GFX1150-NEXT: v_fmac_f16_e32 v7, v6, v5 -; GFX1150-NEXT: v_cvt_f32_f16_e32 v5, v2 -; GFX1150-NEXT: v_lshrrev_b32_e32 v6, 16, v1 -; GFX1150-NEXT: v_rcp_f32_e32 v5, v5 -; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fma_mixlo_f16 v5, v0, v5, 0 op_sel_hi:[1,0,0] -; GFX1150-NEXT: v_div_fixup_f16 v5, v5, v2, v0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_trunc_f16_e32 v5, v5 -; GFX1150-NEXT: v_xor_b32_e32 v5, 0x8000, v5 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) -; GFX1150-NEXT: v_fma_f16 v0, v5, v2, v0 +; GFX1150-NEXT: v_fma_f16 v5, v6, v5, v7 +; GFX1150-NEXT: v_cvt_f32_f16_e32 v6, v2 +; GFX1150-NEXT: v_bfi_b32 v5, 0x7fff, v5, v7 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: v_rcp_f32_e32 v6, v6 +; GFX1150-NEXT: v_fma_mixlo_f16 v6, v0, v6, 0 op_sel_hi:[1,0,0] +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_div_fixup_f16 v6, v6, v2, v0 +; GFX1150-NEXT: v_trunc_f16_e32 v6, v6 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_xor_b32_e32 v6, 0x8000, v6 +; GFX1150-NEXT: v_fma_f16 v2, v6, v2, v0 +; GFX1150-NEXT: v_lshrrev_b32_e32 v6, 16, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_bfi_b32 v0, 0x7fff, v2, v0 ; GFX1150-NEXT: v_lshrrev_b32_e32 v2, 16, v3 -; GFX1150-NEXT: v_pack_b32_f16 v0, v0, v7 +; GFX1150-NEXT: v_pack_b32_f16 v0, v0, v5 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_cvt_f32_f16_e32 v5, v2 ; GFX1150-NEXT: v_rcp_f32_e32 v5, v5 @@ -2596,19 +2725,22 @@ define amdgpu_kernel void @frem_v4f16(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_trunc_f16_e32 v5, v5 ; GFX1150-NEXT: v_xor_b32_e32 v5, 0x8000, v5 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fmac_f16_e32 v6, v5, v2 -; GFX1150-NEXT: v_cvt_f32_f16_e32 v2, v3 -; GFX1150-NEXT: v_rcp_f32_e32 v2, v2 -; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fma_mixlo_f16 v2, v1, v2, 0 op_sel_hi:[1,0,0] -; GFX1150-NEXT: v_div_fixup_f16 v2, v2, v3, v1 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_fma_f16 v2, v5, v2, v6 +; GFX1150-NEXT: v_cvt_f32_f16_e32 v5, v3 +; GFX1150-NEXT: v_bfi_b32 v2, 0x7fff, v2, v6 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: v_rcp_f32_e32 v5, v5 +; GFX1150-NEXT: v_fma_mixlo_f16 v5, v1, v5, 0 op_sel_hi:[1,0,0] ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_trunc_f16_e32 v2, v2 -; GFX1150-NEXT: v_xor_b32_e32 v2, 0x8000, v2 +; GFX1150-NEXT: v_div_fixup_f16 v5, v5, v3, v1 +; GFX1150-NEXT: v_trunc_f16_e32 v5, v5 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_xor_b32_e32 v5, 0x8000, v5 +; GFX1150-NEXT: v_fma_f16 v3, v5, v3, v1 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fmac_f16_e32 v1, v2, v3 -; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v6 +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fff, v3, v1 +; GFX1150-NEXT: v_pack_b32_f16 v1, v1, v2 ; GFX1150-NEXT: global_store_b64 v4, v[0:1], s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2655,7 +2787,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 ; SI-NEXT: v_div_fixup_f32 v4, v4, v3, v1 ; SI-NEXT: v_trunc_f32_e32 v4, v4 -; SI-NEXT: v_fma_f32 v1, -v4, v3, v1 +; SI-NEXT: v_fma_f32 v3, -v4, v3, v1 +; SI-NEXT: s_brev_b32 s6, -2 +; SI-NEXT: v_bfi_b32 v1, s6, v3, v1 ; SI-NEXT: v_div_scale_f32 v3, vcc, v0, v2, v0 ; SI-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v0 ; SI-NEXT: v_rcp_f32_e32 v5, v4 @@ -2670,7 +2804,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v3, v3, v5, v6 ; SI-NEXT: v_div_fixup_f32 v3, v3, v2, v0 ; SI-NEXT: v_trunc_f32_e32 v3, v3 -; SI-NEXT: v_fma_f32 v0, -v3, v2, v0 +; SI-NEXT: v_fma_f32 v2, -v3, v2, v0 +; SI-NEXT: v_bfi_b32 v0, s6, v2, v0 ; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -2691,6 +2826,7 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s11, s3 ; CI-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0 ; CI-NEXT: buffer_load_dwordx2 v[2:3], off, s[8:11], 0 offset:32 +; CI-NEXT: s_brev_b32 s6, -2 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_div_scale_f32 v5, s[4:5], v3, v3, v1 ; CI-NEXT: v_div_scale_f32 v4, vcc, v1, v3, v1 @@ -2706,8 +2842,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v4, v4, v6, v7 ; CI-NEXT: v_div_fixup_f32 v4, v4, v3, v1 ; CI-NEXT: v_trunc_f32_e32 v4, v4 -; CI-NEXT: v_fma_f32 v1, -v4, v3, v1 +; CI-NEXT: v_fma_f32 v3, -v4, v3, v1 ; CI-NEXT: v_div_scale_f32 v4, s[4:5], v2, v2, v0 +; CI-NEXT: v_bfi_b32 v1, s6, v3, v1 ; CI-NEXT: v_div_scale_f32 v3, vcc, v0, v2, v0 ; CI-NEXT: v_rcp_f32_e32 v5, v4 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -2721,7 +2858,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v3, v3, v5, v6 ; CI-NEXT: v_div_fixup_f32 v3, v3, v2, v0 ; CI-NEXT: v_trunc_f32_e32 v3, v3 -; CI-NEXT: v_fma_f32 v0, -v3, v2, v0 +; CI-NEXT: v_fma_f32 v2, -v3, v2, v0 +; CI-NEXT: v_bfi_b32 v0, s6, v2, v0 ; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 ; CI-NEXT: s_endpgm ; @@ -2729,6 +2867,7 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_brev_b32 s2, -2 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v2, s6 ; VI-NEXT: s_add_u32 s0, s0, 32 @@ -2755,8 +2894,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f32 v6, v6, v8, v9 ; VI-NEXT: v_div_fixup_f32 v6, v6, v5, v3 ; VI-NEXT: v_trunc_f32_e32 v6, v6 -; VI-NEXT: v_fma_f32 v3, -v6, v5, v3 +; VI-NEXT: v_fma_f32 v5, -v6, v5, v3 ; VI-NEXT: v_div_scale_f32 v6, s[0:1], v4, v4, v2 +; VI-NEXT: v_bfi_b32 v3, s2, v5, v3 ; VI-NEXT: v_div_scale_f32 v5, vcc, v2, v4, v2 ; VI-NEXT: v_rcp_f32_e32 v7, v6 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -2770,7 +2910,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f32 v5, v5, v7, v8 ; VI-NEXT: v_div_fixup_f32 v5, v5, v4, v2 ; VI-NEXT: v_trunc_f32_e32 v5, v5 -; VI-NEXT: v_fma_f32 v2, -v5, v4, v2 +; VI-NEXT: v_fma_f32 v4, -v5, v4, v2 +; VI-NEXT: v_bfi_b32 v2, s2, v4, v2 ; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3] ; VI-NEXT: s_endpgm ; @@ -2782,6 +2923,7 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx2 v[0:1], v4, s[6:7] ; GFX9-NEXT: global_load_dwordx2 v[2:3], v4, s[2:3] offset:32 +; GFX9-NEXT: s_brev_b32 s2, -2 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_div_scale_f32 v6, s[0:1], v3, v3, v1 ; GFX9-NEXT: v_div_scale_f32 v5, vcc, v1, v3, v1 @@ -2797,8 +2939,9 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f32 v5, v5, v7, v8 ; GFX9-NEXT: v_div_fixup_f32 v5, v5, v3, v1 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 -; GFX9-NEXT: v_fma_f32 v1, -v5, v3, v1 +; GFX9-NEXT: v_fma_f32 v3, -v5, v3, v1 ; GFX9-NEXT: v_div_scale_f32 v5, s[0:1], v2, v2, v0 +; GFX9-NEXT: v_bfi_b32 v1, s2, v3, v1 ; GFX9-NEXT: v_div_scale_f32 v3, vcc, v0, v2, v0 ; GFX9-NEXT: v_rcp_f32_e32 v6, v5 ; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -2812,7 +2955,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f32 v3, v3, v6, v7 ; GFX9-NEXT: v_div_fixup_f32 v3, v3, v2, v0 ; GFX9-NEXT: v_trunc_f32_e32 v3, v3 -; GFX9-NEXT: v_fma_f32 v0, -v3, v2, v0 +; GFX9-NEXT: v_fma_f32 v2, -v3, v2, v0 +; GFX9-NEXT: v_bfi_b32 v0, s2, v2, v0 ; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5] ; GFX9-NEXT: s_endpgm ; @@ -2841,10 +2985,11 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f32 v5, v5, v7, v8 ; GFX10-NEXT: v_div_fixup_f32 v5, v5, v3, v1 ; GFX10-NEXT: v_trunc_f32_e32 v5, v5 -; GFX10-NEXT: v_fma_f32 v1, -v5, v3, v1 +; GFX10-NEXT: v_fma_f32 v3, -v5, v3, v1 ; GFX10-NEXT: v_div_scale_f32 v5, s0, v2, v2, v0 -; GFX10-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v3, v1 ; GFX10-NEXT: v_rcp_f32_e32 v6, v5 +; GFX10-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0 ; GFX10-NEXT: s_denorm_mode 15 ; GFX10-NEXT: v_fma_f32 v7, -v5, v6, 1.0 ; GFX10-NEXT: v_fmac_f32_e32 v6, v7, v6 @@ -2856,7 +3001,8 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f32 v3, v3, v6, v7 ; GFX10-NEXT: v_div_fixup_f32 v3, v3, v2, v0 ; GFX10-NEXT: v_trunc_f32_e32 v3, v3 -; GFX10-NEXT: v_fma_f32 v0, -v3, v2, v0 +; GFX10-NEXT: v_fma_f32 v2, -v3, v2, v0 +; GFX10-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0 ; GFX10-NEXT: global_store_dwordx2 v4, v[0:1], s[4:5] ; GFX10-NEXT: s_endpgm ; @@ -2891,28 +3037,31 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_div_fixup_f32 v5, v5, v3, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f32_e32 v5, v5 -; GFX11-NEXT: v_fma_f32 v1, -v5, v3, v1 +; GFX11-NEXT: v_fma_f32 v3, -v5, v3, v1 ; GFX11-NEXT: v_div_scale_f32 v5, null, v2, v2, v0 -; GFX11-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v3, v1 ; GFX11-NEXT: v_rcp_f32_e32 v6, v5 +; GFX11-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0 ; GFX11-NEXT: s_denorm_mode 15 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_f32 v7, -v5, v6, 1.0 -; GFX11-NEXT: v_fmac_f32_e32 v6, v7, v6 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v6, v7, v6 ; GFX11-NEXT: v_mul_f32_e32 v7, v3, v6 -; GFX11-NEXT: v_fma_f32 v8, -v5, v7, v3 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fma_f32 v8, -v5, v7, v3 ; GFX11-NEXT: v_fmac_f32_e32 v7, v8, v6 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_fma_f32 v3, -v5, v7, v3 ; GFX11-NEXT: s_denorm_mode 12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_div_fmas_f32 v3, v3, v6, v7 -; GFX11-NEXT: v_div_fixup_f32 v3, v3, v2, v0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_div_fixup_f32 v3, v3, v2, v0 ; GFX11-NEXT: v_trunc_f32_e32 v3, v3 -; GFX11-NEXT: v_fma_f32 v0, -v3, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fma_f32 v2, -v3, v2, v0 +; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0 ; GFX11-NEXT: global_store_b64 v4, v[0:1], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -2950,29 +3099,32 @@ define amdgpu_kernel void @frem_v2f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: v_trunc_f32_e32 v5, v5 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v5, 0x80000000, v5 -; GFX1150-NEXT: v_fma_f32 v1, v5, v3, v1 +; GFX1150-NEXT: v_fma_f32 v3, v5, v3, v1 ; GFX1150-NEXT: v_div_scale_f32 v5, null, v2, v2, v0 -; GFX1150-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v3, v1 ; GFX1150-NEXT: v_rcp_f32_e32 v6, v5 +; GFX1150-NEXT: v_div_scale_f32 v3, vcc_lo, v0, v2, v0 ; GFX1150-NEXT: s_denorm_mode 15 +; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f32 v7, -v5, v6, 1.0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fmac_f32_e32 v6, v7, v6 -; GFX1150-NEXT: v_mul_f32_e32 v7, v3, v6 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_mul_f32_e32 v7, v3, v6 ; GFX1150-NEXT: v_fma_f32 v8, -v5, v7, v3 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fmac_f32_e32 v7, v8, v6 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f32 v3, -v5, v7, v3 ; GFX1150-NEXT: s_denorm_mode 12 -; GFX1150-NEXT: v_div_fmas_f32 v3, v3, v6, v7 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_div_fmas_f32 v3, v3, v6, v7 ; GFX1150-NEXT: v_div_fixup_f32 v3, v3, v2, v0 -; GFX1150-NEXT: v_trunc_f32_e32 v3, v3 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_trunc_f32_e32 v3, v3 ; GFX1150-NEXT: v_xor_b32_e32 v3, 0x80000000, v3 -; GFX1150-NEXT: v_fmac_f32_e32 v0, v3, v2 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_fma_f32 v2, v3, v2, v0 +; GFX1150-NEXT: v_bfi_b32 v0, 0x7fffffff, v2, v0 ; GFX1150-NEXT: global_store_b64 v4, v[0:1], s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -3019,7 +3171,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v8, v8, v10, v11 ; SI-NEXT: v_div_fixup_f32 v8, v8, v7, v3 ; SI-NEXT: v_trunc_f32_e32 v8, v8 -; SI-NEXT: v_fma_f32 v3, -v8, v7, v3 +; SI-NEXT: v_fma_f32 v7, -v8, v7, v3 +; SI-NEXT: s_brev_b32 s6, -2 +; SI-NEXT: v_bfi_b32 v3, s6, v7, v3 ; SI-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2 ; SI-NEXT: v_div_scale_f32 v8, s[4:5], v6, v6, v2 ; SI-NEXT: v_rcp_f32_e32 v9, v8 @@ -3034,7 +3188,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v7, v7, v9, v10 ; SI-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; SI-NEXT: v_trunc_f32_e32 v7, v7 -; SI-NEXT: v_fma_f32 v2, -v7, v6, v2 +; SI-NEXT: v_fma_f32 v6, -v7, v6, v2 +; SI-NEXT: v_bfi_b32 v2, s6, v6, v2 ; SI-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1 ; SI-NEXT: v_div_scale_f32 v7, s[4:5], v5, v5, v1 ; SI-NEXT: v_rcp_f32_e32 v8, v7 @@ -3049,7 +3204,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v6, v6, v8, v9 ; SI-NEXT: v_div_fixup_f32 v6, v6, v5, v1 ; SI-NEXT: v_trunc_f32_e32 v6, v6 -; SI-NEXT: v_fma_f32 v1, -v6, v5, v1 +; SI-NEXT: v_fma_f32 v5, -v6, v5, v1 +; SI-NEXT: v_bfi_b32 v1, s6, v5, v1 ; SI-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0 ; SI-NEXT: v_div_scale_f32 v6, s[4:5], v4, v4, v0 ; SI-NEXT: v_rcp_f32_e32 v7, v6 @@ -3064,7 +3220,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_div_fmas_f32 v5, v5, v7, v8 ; SI-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; SI-NEXT: v_trunc_f32_e32 v5, v5 -; SI-NEXT: v_fma_f32 v0, -v5, v4, v0 +; SI-NEXT: v_fma_f32 v4, -v5, v4, v0 +; SI-NEXT: v_bfi_b32 v0, s6, v4, v0 ; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; SI-NEXT: s_endpgm ; @@ -3085,6 +3242,7 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s11, s3 ; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0 ; CI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:64 +; CI-NEXT: s_brev_b32 s6, -2 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_div_scale_f32 v9, s[4:5], v7, v7, v3 ; CI-NEXT: v_div_scale_f32 v8, vcc, v3, v7, v3 @@ -3100,8 +3258,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v8, v8, v10, v11 ; CI-NEXT: v_div_fixup_f32 v8, v8, v7, v3 ; CI-NEXT: v_trunc_f32_e32 v8, v8 -; CI-NEXT: v_fma_f32 v3, -v8, v7, v3 +; CI-NEXT: v_fma_f32 v7, -v8, v7, v3 ; CI-NEXT: v_div_scale_f32 v8, s[4:5], v6, v6, v2 +; CI-NEXT: v_bfi_b32 v3, s6, v7, v3 ; CI-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2 ; CI-NEXT: v_rcp_f32_e32 v9, v8 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3115,8 +3274,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v7, v7, v9, v10 ; CI-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; CI-NEXT: v_trunc_f32_e32 v7, v7 -; CI-NEXT: v_fma_f32 v2, -v7, v6, v2 +; CI-NEXT: v_fma_f32 v6, -v7, v6, v2 ; CI-NEXT: v_div_scale_f32 v7, s[4:5], v5, v5, v1 +; CI-NEXT: v_bfi_b32 v2, s6, v6, v2 ; CI-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1 ; CI-NEXT: v_rcp_f32_e32 v8, v7 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3130,8 +3290,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v6, v6, v8, v9 ; CI-NEXT: v_div_fixup_f32 v6, v6, v5, v1 ; CI-NEXT: v_trunc_f32_e32 v6, v6 -; CI-NEXT: v_fma_f32 v1, -v6, v5, v1 +; CI-NEXT: v_fma_f32 v5, -v6, v5, v1 ; CI-NEXT: v_div_scale_f32 v6, s[4:5], v4, v4, v0 +; CI-NEXT: v_bfi_b32 v1, s6, v5, v1 ; CI-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0 ; CI-NEXT: v_rcp_f32_e32 v7, v6 ; CI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3145,7 +3306,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f32 v5, v5, v7, v8 ; CI-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; CI-NEXT: v_trunc_f32_e32 v5, v5 -; CI-NEXT: v_fma_f32 v0, -v5, v4, v0 +; CI-NEXT: v_fma_f32 v4, -v5, v4, v0 +; CI-NEXT: v_bfi_b32 v0, s6, v4, v0 ; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 ; CI-NEXT: s_endpgm ; @@ -3153,6 +3315,7 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_brev_b32 s2, -2 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s6 ; VI-NEXT: s_add_u32 s0, s0, 64 @@ -3179,8 +3342,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f32 v10, v10, v12, v13 ; VI-NEXT: v_div_fixup_f32 v10, v10, v7, v3 ; VI-NEXT: v_trunc_f32_e32 v10, v10 -; VI-NEXT: v_fma_f32 v3, -v10, v7, v3 +; VI-NEXT: v_fma_f32 v7, -v10, v7, v3 ; VI-NEXT: v_div_scale_f32 v10, s[0:1], v6, v6, v2 +; VI-NEXT: v_bfi_b32 v3, s2, v7, v3 ; VI-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2 ; VI-NEXT: v_rcp_f32_e32 v11, v10 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3194,8 +3358,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f32 v7, v7, v11, v12 ; VI-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; VI-NEXT: v_trunc_f32_e32 v7, v7 -; VI-NEXT: v_fma_f32 v2, -v7, v6, v2 +; VI-NEXT: v_fma_f32 v6, -v7, v6, v2 ; VI-NEXT: v_div_scale_f32 v7, s[0:1], v5, v5, v1 +; VI-NEXT: v_bfi_b32 v2, s2, v6, v2 ; VI-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1 ; VI-NEXT: v_rcp_f32_e32 v10, v7 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3209,8 +3374,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f32 v6, v6, v10, v11 ; VI-NEXT: v_div_fixup_f32 v6, v6, v5, v1 ; VI-NEXT: v_trunc_f32_e32 v6, v6 -; VI-NEXT: v_fma_f32 v1, -v6, v5, v1 +; VI-NEXT: v_fma_f32 v5, -v6, v5, v1 ; VI-NEXT: v_div_scale_f32 v6, s[0:1], v4, v4, v0 +; VI-NEXT: v_bfi_b32 v1, s2, v5, v1 ; VI-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0 ; VI-NEXT: v_rcp_f32_e32 v7, v6 ; VI-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3224,7 +3390,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f32 v5, v5, v7, v10 ; VI-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; VI-NEXT: v_trunc_f32_e32 v5, v5 -; VI-NEXT: v_fma_f32 v0, -v5, v4, v0 +; VI-NEXT: v_fma_f32 v4, -v5, v4, v0 +; VI-NEXT: v_bfi_b32 v0, s2, v4, v0 ; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3] ; VI-NEXT: s_endpgm ; @@ -3236,6 +3403,7 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[0:3], v8, s[6:7] ; GFX9-NEXT: global_load_dwordx4 v[4:7], v8, s[2:3] offset:64 +; GFX9-NEXT: s_brev_b32 s2, -2 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_div_scale_f32 v10, s[0:1], v7, v7, v3 ; GFX9-NEXT: v_div_scale_f32 v9, vcc, v3, v7, v3 @@ -3251,8 +3419,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f32 v9, v9, v11, v12 ; GFX9-NEXT: v_div_fixup_f32 v9, v9, v7, v3 ; GFX9-NEXT: v_trunc_f32_e32 v9, v9 -; GFX9-NEXT: v_fma_f32 v3, -v9, v7, v3 +; GFX9-NEXT: v_fma_f32 v7, -v9, v7, v3 ; GFX9-NEXT: v_div_scale_f32 v9, s[0:1], v6, v6, v2 +; GFX9-NEXT: v_bfi_b32 v3, s2, v7, v3 ; GFX9-NEXT: v_div_scale_f32 v7, vcc, v2, v6, v2 ; GFX9-NEXT: v_rcp_f32_e32 v10, v9 ; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3266,8 +3435,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f32 v7, v7, v10, v11 ; GFX9-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; GFX9-NEXT: v_trunc_f32_e32 v7, v7 -; GFX9-NEXT: v_fma_f32 v2, -v7, v6, v2 +; GFX9-NEXT: v_fma_f32 v6, -v7, v6, v2 ; GFX9-NEXT: v_div_scale_f32 v7, s[0:1], v5, v5, v1 +; GFX9-NEXT: v_bfi_b32 v2, s2, v6, v2 ; GFX9-NEXT: v_div_scale_f32 v6, vcc, v1, v5, v1 ; GFX9-NEXT: v_rcp_f32_e32 v9, v7 ; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3281,8 +3451,9 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f32 v6, v6, v9, v10 ; GFX9-NEXT: v_div_fixup_f32 v6, v6, v5, v1 ; GFX9-NEXT: v_trunc_f32_e32 v6, v6 -; GFX9-NEXT: v_fma_f32 v1, -v6, v5, v1 +; GFX9-NEXT: v_fma_f32 v5, -v6, v5, v1 ; GFX9-NEXT: v_div_scale_f32 v6, s[0:1], v4, v4, v0 +; GFX9-NEXT: v_bfi_b32 v1, s2, v5, v1 ; GFX9-NEXT: v_div_scale_f32 v5, vcc, v0, v4, v0 ; GFX9-NEXT: v_rcp_f32_e32 v7, v6 ; GFX9-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3 @@ -3296,7 +3467,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f32 v5, v5, v7, v9 ; GFX9-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; GFX9-NEXT: v_trunc_f32_e32 v5, v5 -; GFX9-NEXT: v_fma_f32 v0, -v5, v4, v0 +; GFX9-NEXT: v_fma_f32 v4, -v5, v4, v0 +; GFX9-NEXT: v_bfi_b32 v0, s2, v4, v0 ; GFX9-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] ; GFX9-NEXT: s_endpgm ; @@ -3325,10 +3497,11 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f32 v9, v9, v11, v12 ; GFX10-NEXT: v_div_fixup_f32 v9, v9, v7, v3 ; GFX10-NEXT: v_trunc_f32_e32 v9, v9 -; GFX10-NEXT: v_fma_f32 v3, -v9, v7, v3 +; GFX10-NEXT: v_fma_f32 v7, -v9, v7, v3 ; GFX10-NEXT: v_div_scale_f32 v9, s0, v6, v6, v2 -; GFX10-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2 +; GFX10-NEXT: v_bfi_b32 v3, 0x7fffffff, v7, v3 ; GFX10-NEXT: v_rcp_f32_e32 v10, v9 +; GFX10-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2 ; GFX10-NEXT: s_denorm_mode 15 ; GFX10-NEXT: v_fma_f32 v11, -v9, v10, 1.0 ; GFX10-NEXT: v_fmac_f32_e32 v10, v11, v10 @@ -3340,10 +3513,11 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f32 v7, v7, v10, v11 ; GFX10-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; GFX10-NEXT: v_trunc_f32_e32 v7, v7 -; GFX10-NEXT: v_fma_f32 v2, -v7, v6, v2 +; GFX10-NEXT: v_fma_f32 v6, -v7, v6, v2 ; GFX10-NEXT: v_div_scale_f32 v7, s0, v5, v5, v1 -; GFX10-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1 +; GFX10-NEXT: v_bfi_b32 v2, 0x7fffffff, v6, v2 ; GFX10-NEXT: v_rcp_f32_e32 v9, v7 +; GFX10-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1 ; GFX10-NEXT: s_denorm_mode 15 ; GFX10-NEXT: v_fma_f32 v10, -v7, v9, 1.0 ; GFX10-NEXT: v_fmac_f32_e32 v9, v10, v9 @@ -3355,10 +3529,11 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f32 v6, v6, v9, v10 ; GFX10-NEXT: v_div_fixup_f32 v6, v6, v5, v1 ; GFX10-NEXT: v_trunc_f32_e32 v6, v6 -; GFX10-NEXT: v_fma_f32 v1, -v6, v5, v1 +; GFX10-NEXT: v_fma_f32 v5, -v6, v5, v1 ; GFX10-NEXT: v_div_scale_f32 v6, s0, v4, v4, v0 -; GFX10-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0 +; GFX10-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1 ; GFX10-NEXT: v_rcp_f32_e32 v7, v6 +; GFX10-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0 ; GFX10-NEXT: s_denorm_mode 15 ; GFX10-NEXT: v_fma_f32 v9, -v6, v7, 1.0 ; GFX10-NEXT: v_fmac_f32_e32 v7, v9, v7 @@ -3370,7 +3545,8 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f32 v5, v5, v7, v9 ; GFX10-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; GFX10-NEXT: v_trunc_f32_e32 v5, v5 -; GFX10-NEXT: v_fma_f32 v0, -v5, v4, v0 +; GFX10-NEXT: v_fma_f32 v4, -v5, v4, v0 +; GFX10-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0 ; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[4:5] ; GFX10-NEXT: s_endpgm ; @@ -3405,32 +3581,35 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_div_fixup_f32 v9, v9, v7, v3 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f32_e32 v9, v9 -; GFX11-NEXT: v_fma_f32 v3, -v9, v7, v3 +; GFX11-NEXT: v_fma_f32 v7, -v9, v7, v3 ; GFX11-NEXT: v_div_scale_f32 v9, null, v6, v6, v2 -; GFX11-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_bfi_b32 v3, 0x7fffffff, v7, v3 ; GFX11-NEXT: v_rcp_f32_e32 v10, v9 +; GFX11-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2 ; GFX11-NEXT: s_denorm_mode 15 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_f32 v11, -v9, v10, 1.0 -; GFX11-NEXT: v_fmac_f32_e32 v10, v11, v10 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v10, v11, v10 ; GFX11-NEXT: v_mul_f32_e32 v11, v7, v10 -; GFX11-NEXT: v_fma_f32 v12, -v9, v11, v7 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fma_f32 v12, -v9, v11, v7 ; GFX11-NEXT: v_fmac_f32_e32 v11, v12, v10 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_fma_f32 v7, -v9, v11, v7 ; GFX11-NEXT: s_denorm_mode 12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_div_fmas_f32 v7, v7, v10, v11 -; GFX11-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_div_fixup_f32 v7, v7, v6, v2 ; GFX11-NEXT: v_trunc_f32_e32 v7, v7 -; GFX11-NEXT: v_fma_f32 v2, -v7, v6, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_fma_f32 v6, -v7, v6, v2 ; GFX11-NEXT: v_div_scale_f32 v7, null, v5, v5, v1 -; GFX11-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v2, 0x7fffffff, v6, v2 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_rcp_f32_e32 v9, v7 +; GFX11-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1 ; GFX11-NEXT: s_denorm_mode 15 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_f32 v10, -v7, v9, 1.0 @@ -3447,28 +3626,31 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_div_fixup_f32 v6, v6, v5, v1 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f32_e32 v6, v6 -; GFX11-NEXT: v_fma_f32 v1, -v6, v5, v1 +; GFX11-NEXT: v_fma_f32 v5, -v6, v5, v1 ; GFX11-NEXT: v_div_scale_f32 v6, null, v4, v4, v0 -; GFX11-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_3) | instid1(VALU_DEP_1) +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1 ; GFX11-NEXT: v_rcp_f32_e32 v7, v6 +; GFX11-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0 ; GFX11-NEXT: s_denorm_mode 15 ; GFX11-NEXT: s_waitcnt_depctr 0xfff ; GFX11-NEXT: v_fma_f32 v9, -v6, v7, 1.0 -; GFX11-NEXT: v_fmac_f32_e32 v7, v9, v7 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fmac_f32_e32 v7, v9, v7 ; GFX11-NEXT: v_mul_f32_e32 v9, v5, v7 -; GFX11-NEXT: v_fma_f32 v10, -v6, v9, v5 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fma_f32 v10, -v6, v9, v5 ; GFX11-NEXT: v_fmac_f32_e32 v9, v10, v7 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_fma_f32 v5, -v6, v9, v5 ; GFX11-NEXT: s_denorm_mode 12 -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_div_fmas_f32 v5, v5, v7, v9 -; GFX11-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_div_fixup_f32 v5, v5, v4, v0 ; GFX11-NEXT: v_trunc_f32_e32 v5, v5 -; GFX11-NEXT: v_fma_f32 v0, -v5, v4, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fma_f32 v4, -v5, v4, v0 +; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0 ; GFX11-NEXT: global_store_b128 v8, v[0:3], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -3506,33 +3688,36 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: v_trunc_f32_e32 v9, v9 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v9, 0x80000000, v9 -; GFX1150-NEXT: v_fma_f32 v3, v9, v7, v3 +; GFX1150-NEXT: v_fma_f32 v7, v9, v7, v3 ; GFX1150-NEXT: v_div_scale_f32 v9, null, v6, v6, v2 -; GFX1150-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_bfi_b32 v3, 0x7fffffff, v7, v3 ; GFX1150-NEXT: v_rcp_f32_e32 v10, v9 +; GFX1150-NEXT: v_div_scale_f32 v7, vcc_lo, v2, v6, v2 ; GFX1150-NEXT: s_denorm_mode 15 +; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f32 v11, -v9, v10, 1.0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fmac_f32_e32 v10, v11, v10 -; GFX1150-NEXT: v_mul_f32_e32 v11, v7, v10 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_mul_f32_e32 v11, v7, v10 ; GFX1150-NEXT: v_fma_f32 v12, -v9, v11, v7 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fmac_f32_e32 v11, v12, v10 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f32 v7, -v9, v11, v7 ; GFX1150-NEXT: s_denorm_mode 12 -; GFX1150-NEXT: v_div_fmas_f32 v7, v7, v10, v11 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_div_fmas_f32 v7, v7, v10, v11 ; GFX1150-NEXT: v_div_fixup_f32 v7, v7, v6, v2 -; GFX1150-NEXT: v_trunc_f32_e32 v7, v7 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_trunc_f32_e32 v7, v7 ; GFX1150-NEXT: v_xor_b32_e32 v7, 0x80000000, v7 -; GFX1150-NEXT: v_fma_f32 v2, v7, v6, v2 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_fma_f32 v6, v7, v6, v2 ; GFX1150-NEXT: v_div_scale_f32 v7, null, v5, v5, v1 -; GFX1150-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: v_bfi_b32 v2, 0x7fffffff, v6, v2 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(TRANS32_DEP_1) ; GFX1150-NEXT: v_rcp_f32_e32 v9, v7 +; GFX1150-NEXT: v_div_scale_f32 v6, vcc_lo, v1, v5, v1 ; GFX1150-NEXT: s_denorm_mode 15 ; GFX1150-NEXT: v_fma_f32 v10, -v7, v9, 1.0 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) @@ -3550,29 +3735,32 @@ define amdgpu_kernel void @frem_v4f32(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: v_trunc_f32_e32 v6, v6 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_xor_b32_e32 v6, 0x80000000, v6 -; GFX1150-NEXT: v_fma_f32 v1, v6, v5, v1 +; GFX1150-NEXT: v_fma_f32 v5, v6, v5, v1 ; GFX1150-NEXT: v_div_scale_f32 v6, null, v4, v4, v0 -; GFX1150-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX1150-NEXT: v_bfi_b32 v1, 0x7fffffff, v5, v1 ; GFX1150-NEXT: v_rcp_f32_e32 v7, v6 +; GFX1150-NEXT: v_div_scale_f32 v5, vcc_lo, v0, v4, v0 ; GFX1150-NEXT: s_denorm_mode 15 +; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f32 v9, -v6, v7, 1.0 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fmac_f32_e32 v7, v9, v7 -; GFX1150-NEXT: v_mul_f32_e32 v9, v5, v7 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_mul_f32_e32 v9, v5, v7 ; GFX1150-NEXT: v_fma_f32 v10, -v6, v9, v5 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fmac_f32_e32 v9, v10, v7 -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f32 v5, -v6, v9, v5 ; GFX1150-NEXT: s_denorm_mode 12 -; GFX1150-NEXT: v_div_fmas_f32 v5, v5, v7, v9 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_div_fmas_f32 v5, v5, v7, v9 ; GFX1150-NEXT: v_div_fixup_f32 v5, v5, v4, v0 -; GFX1150-NEXT: v_trunc_f32_e32 v5, v5 ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_trunc_f32_e32 v5, v5 ; GFX1150-NEXT: v_xor_b32_e32 v5, 0x80000000, v5 -; GFX1150-NEXT: v_fmac_f32_e32 v0, v5, v4 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_fma_f32 v4, v5, v4, v0 +; GFX1150-NEXT: v_bfi_b32 v0, 0x7fffffff, v4, v0 ; GFX1150-NEXT: global_store_b128 v8, v[0:3], s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) @@ -3640,43 +3828,46 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; SI-NEXT: v_mov_b32_e32 v11, s8 ; SI-NEXT: v_cndmask_b32_e32 v9, v9, v11, vcc ; SI-NEXT: v_cndmask_b32_e32 v8, v10, v8, vcc -; SI-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3] -; SI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1] -; SI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] -; SI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; SI-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3] +; SI-NEXT: s_brev_b32 s8, -2 +; SI-NEXT: v_bfi_b32 v7, s8, v7, v3 +; SI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[4:5], v[4:5], v[0:1] +; SI-NEXT: v_rcp_f64_e32 v[8:9], v[2:3] +; SI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] -; SI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; SI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; SI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; SI-NEXT: v_div_scale_f64 v[10:11], s[0:1], v[0:1], v[4:5], v[0:1] ; SI-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9] -; SI-NEXT: v_fma_f64 v[14:15], -v[6:7], v[12:13], v[10:11] -; SI-NEXT: v_cmp_eq_u32_e32 vcc, v5, v7 +; SI-NEXT: v_fma_f64 v[14:15], -v[2:3], v[12:13], v[10:11] +; SI-NEXT: v_cmp_eq_u32_e32 vcc, v5, v3 ; SI-NEXT: v_cmp_eq_u32_e64 s[0:1], v1, v11 ; SI-NEXT: s_xor_b64 vcc, s[0:1], vcc ; SI-NEXT: s_nop 1 -; SI-NEXT: v_div_fmas_f64 v[6:7], v[14:15], v[8:9], v[12:13] -; SI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] -; SI-NEXT: v_readfirstlane_b32 s8, v7 -; SI-NEXT: s_bfe_u32 s0, s8, 0xb0014 -; SI-NEXT: s_add_i32 s9, s0, 0xfffffc01 -; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s9 +; SI-NEXT: v_div_fmas_f64 v[2:3], v[14:15], v[8:9], v[12:13] +; SI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; SI-NEXT: v_readfirstlane_b32 s9, v3 +; SI-NEXT: s_bfe_u32 s0, s9, 0xb0014 +; SI-NEXT: s_add_i32 s10, s0, 0xfffffc01 +; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s10 ; SI-NEXT: v_not_b32_e32 v8, s0 -; SI-NEXT: v_and_b32_e32 v8, v6, v8 +; SI-NEXT: v_and_b32_e32 v8, v2, v8 ; SI-NEXT: v_not_b32_e32 v9, s1 -; SI-NEXT: v_and_b32_e32 v7, v7, v9 -; SI-NEXT: s_and_b32 s0, s8, 0x80000000 -; SI-NEXT: s_cmp_lt_i32 s9, 0 +; SI-NEXT: v_and_b32_e32 v3, v3, v9 +; SI-NEXT: s_and_b32 s0, s9, 0x80000000 +; SI-NEXT: s_cmp_lt_i32 s10, 0 ; SI-NEXT: s_cselect_b64 vcc, -1, 0 ; SI-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc ; SI-NEXT: v_mov_b32_e32 v9, s0 -; SI-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc -; SI-NEXT: s_cmp_gt_i32 s9, 51 +; SI-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc +; SI-NEXT: s_cmp_gt_i32 s10, 51 ; SI-NEXT: s_cselect_b64 vcc, -1, 0 -; SI-NEXT: v_mov_b32_e32 v9, s8 -; SI-NEXT: v_cndmask_b32_e32 v7, v7, v9, vcc -; SI-NEXT: v_cndmask_b32_e32 v6, v8, v6, vcc -; SI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0 +; SI-NEXT: v_mov_b32_e32 v9, s9 +; SI-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc +; SI-NEXT: v_cndmask_b32_e32 v2, v8, v2, vcc +; SI-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; SI-NEXT: v_bfi_b32 v5, s8, v5, v1 +; SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[4:7], 0 ; SI-NEXT: s_endpgm ; ; CI-LABEL: frem_v2f64: @@ -3696,6 +3887,7 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: s_mov_b32 s11, s3 ; CI-NEXT: buffer_load_dwordx4 v[0:3], off, s[4:7], 0 ; CI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:64 +; CI-NEXT: s_brev_b32 s6, -2 ; CI-NEXT: s_waitcnt vmcnt(0) ; CI-NEXT: v_div_scale_f64 v[8:9], s[4:5], v[6:7], v[6:7], v[2:3] ; CI-NEXT: v_rcp_f64_e32 v[10:11], v[8:9] @@ -3710,28 +3902,31 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; CI-NEXT: v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15] ; CI-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3] ; CI-NEXT: v_trunc_f64_e32 v[8:9], v[8:9] -; CI-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3] -; CI-NEXT: v_div_scale_f64 v[6:7], s[4:5], v[4:5], v[4:5], v[0:1] -; CI-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] -; CI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; CI-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3] +; CI-NEXT: v_bfi_b32 v7, s6, v7, v3 +; CI-NEXT: v_div_scale_f64 v[2:3], s[4:5], v[4:5], v[4:5], v[0:1] +; CI-NEXT: v_rcp_f64_e32 v[8:9], v[2:3] +; CI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; CI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] -; CI-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; CI-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; CI-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; CI-NEXT: v_div_scale_f64 v[10:11], vcc, v[0:1], v[4:5], v[0:1] ; CI-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9] -; CI-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11] +; CI-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11] ; CI-NEXT: s_nop 1 -; CI-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13] -; CI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] -; CI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; CI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; CI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 +; CI-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13] +; CI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; CI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] +; CI-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; CI-NEXT: v_bfi_b32 v5, s6, v5, v1 +; CI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 ; CI-NEXT: s_endpgm ; ; VI-LABEL: frem_v2f64: ; VI: ; %bb.0: ; VI-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; VI-NEXT: s_brev_b32 s2, -2 ; VI-NEXT: s_waitcnt lgkmcnt(0) ; VI-NEXT: v_mov_b32_e32 v0, s6 ; VI-NEXT: s_add_u32 s0, s0, 64 @@ -3757,22 +3952,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; VI-NEXT: v_div_fmas_f64 v[10:11], v[10:11], v[12:13], v[16:17] ; VI-NEXT: v_div_fixup_f64 v[10:11], v[10:11], v[6:7], v[2:3] ; VI-NEXT: v_trunc_f64_e32 v[10:11], v[10:11] -; VI-NEXT: v_fma_f64 v[2:3], -v[10:11], v[6:7], v[2:3] -; VI-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1] -; VI-NEXT: v_rcp_f64_e32 v[10:11], v[6:7] -; VI-NEXT: v_fma_f64 v[12:13], -v[6:7], v[10:11], 1.0 +; VI-NEXT: v_fma_f64 v[6:7], -v[10:11], v[6:7], v[2:3] +; VI-NEXT: v_bfi_b32 v7, s2, v7, v3 +; VI-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[4:5], v[4:5], v[0:1] +; VI-NEXT: v_rcp_f64_e32 v[10:11], v[2:3] +; VI-NEXT: v_fma_f64 v[12:13], -v[2:3], v[10:11], 1.0 ; VI-NEXT: v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11] -; VI-NEXT: v_fma_f64 v[12:13], -v[6:7], v[10:11], 1.0 +; VI-NEXT: v_fma_f64 v[12:13], -v[2:3], v[10:11], 1.0 ; VI-NEXT: v_fma_f64 v[10:11], v[10:11], v[12:13], v[10:11] ; VI-NEXT: v_div_scale_f64 v[12:13], vcc, v[0:1], v[4:5], v[0:1] ; VI-NEXT: v_mul_f64 v[14:15], v[12:13], v[10:11] -; VI-NEXT: v_fma_f64 v[6:7], -v[6:7], v[14:15], v[12:13] +; VI-NEXT: v_fma_f64 v[2:3], -v[2:3], v[14:15], v[12:13] ; VI-NEXT: s_nop 1 -; VI-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[10:11], v[14:15] -; VI-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] -; VI-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; VI-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; VI-NEXT: flat_store_dwordx4 v[8:9], v[0:3] +; VI-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[10:11], v[14:15] +; VI-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; VI-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] +; VI-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; VI-NEXT: v_bfi_b32 v5, s2, v5, v1 +; VI-NEXT: flat_store_dwordx4 v[8:9], v[4:7] ; VI-NEXT: s_endpgm ; ; GFX9-LABEL: frem_v2f64: @@ -3783,6 +3980,7 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NEXT: global_load_dwordx4 v[0:3], v16, s[6:7] ; GFX9-NEXT: global_load_dwordx4 v[4:7], v16, s[2:3] offset:64 +; GFX9-NEXT: s_brev_b32 s2, -2 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_div_scale_f64 v[8:9], s[0:1], v[6:7], v[6:7], v[2:3] ; GFX9-NEXT: v_rcp_f64_e32 v[10:11], v[8:9] @@ -3797,22 +3995,24 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX9-NEXT: v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15] ; GFX9-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3] ; GFX9-NEXT: v_trunc_f64_e32 v[8:9], v[8:9] -; GFX9-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3] -; GFX9-NEXT: v_div_scale_f64 v[6:7], s[0:1], v[4:5], v[4:5], v[0:1] -; GFX9-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] -; GFX9-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; GFX9-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3] +; GFX9-NEXT: v_bfi_b32 v7, s2, v7, v3 +; GFX9-NEXT: v_div_scale_f64 v[2:3], s[0:1], v[4:5], v[4:5], v[0:1] +; GFX9-NEXT: v_rcp_f64_e32 v[8:9], v[2:3] +; GFX9-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; GFX9-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] -; GFX9-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; GFX9-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; GFX9-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; GFX9-NEXT: v_div_scale_f64 v[10:11], vcc, v[0:1], v[4:5], v[0:1] ; GFX9-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9] -; GFX9-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11] +; GFX9-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11] ; GFX9-NEXT: s_nop 1 -; GFX9-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13] -; GFX9-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] -; GFX9-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; GFX9-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; GFX9-NEXT: global_store_dwordx4 v16, v[0:3], s[4:5] +; GFX9-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13] +; GFX9-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] +; GFX9-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; GFX9-NEXT: v_bfi_b32 v5, s2, v5, v1 +; GFX9-NEXT: global_store_dwordx4 v16, v[4:7], s[4:5] ; GFX9-NEXT: s_endpgm ; ; GFX10-LABEL: frem_v2f64: @@ -3838,21 +4038,23 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX10-NEXT: v_div_fmas_f64 v[8:9], v[8:9], v[10:11], v[14:15] ; GFX10-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3] ; GFX10-NEXT: v_trunc_f64_e32 v[8:9], v[8:9] -; GFX10-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3] -; GFX10-NEXT: v_div_scale_f64 v[6:7], s0, v[4:5], v[4:5], v[0:1] -; GFX10-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] -; GFX10-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; GFX10-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3] +; GFX10-NEXT: v_bfi_b32 v7, 0x7fffffff, v7, v3 +; GFX10-NEXT: v_div_scale_f64 v[2:3], s0, v[4:5], v[4:5], v[0:1] +; GFX10-NEXT: v_rcp_f64_e32 v[8:9], v[2:3] +; GFX10-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; GFX10-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] -; GFX10-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; GFX10-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; GFX10-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; GFX10-NEXT: v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1] ; GFX10-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9] -; GFX10-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11] -; GFX10-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13] -; GFX10-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] -; GFX10-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; GFX10-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; GFX10-NEXT: global_store_dwordx4 v16, v[0:3], s[4:5] +; GFX10-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11] +; GFX10-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13] +; GFX10-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; GFX10-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] +; GFX10-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; GFX10-NEXT: v_bfi_b32 v5, 0x7fffffff, v5, v1 +; GFX10-NEXT: global_store_dwordx4 v16, v[4:7], s[4:5] ; GFX10-NEXT: s_endpgm ; ; GFX11-LABEL: frem_v2f64: @@ -3884,27 +4086,30 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX11-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_trunc_f64_e32 v[8:9], v[8:9] -; GFX11-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3] -; GFX11-NEXT: v_div_scale_f64 v[6:7], null, v[4:5], v[4:5], v[0:1] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] +; GFX11-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_bfi_b32 v7, 0x7fffffff, v7, v3 +; GFX11-NEXT: v_div_scale_f64 v[2:3], null, v[4:5], v[4:5], v[0:1] +; GFX11-NEXT: v_rcp_f64_e32 v[8:9], v[2:3] ; GFX11-NEXT: s_waitcnt_depctr 0xfff -; GFX11-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 -; GFX11-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] +; GFX11-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; GFX11-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] +; GFX11-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; GFX11-NEXT: v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1] -; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX11-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9] -; GFX11-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13] -; GFX11-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] +; GFX11-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11] +; GFX11-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13] ; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX11-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; GFX11-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; GFX11-NEXT: global_store_b128 v16, v[0:3], s[4:5] +; GFX11-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; GFX11-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; GFX11-NEXT: v_bfi_b32 v5, 0x7fffffff, v5, v1 +; GFX11-NEXT: global_store_b128 v16, v[4:7], s[4:5] ; GFX11-NEXT: s_nop 0 ; GFX11-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX11-NEXT: s_endpgm @@ -3937,27 +4142,30 @@ define amdgpu_kernel void @frem_v2f64(ptr addrspace(1) %out, ptr addrspace(1) %i ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_div_fixup_f64 v[8:9], v[8:9], v[6:7], v[2:3] ; GFX1150-NEXT: v_trunc_f64_e32 v[8:9], v[8:9] -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fma_f64 v[2:3], -v[8:9], v[6:7], v[2:3] -; GFX1150-NEXT: v_div_scale_f64 v[6:7], null, v[4:5], v[4:5], v[0:1] -; GFX1150-NEXT: v_rcp_f64_e32 v[8:9], v[6:7] -; GFX1150-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 -; GFX1150-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_fma_f64 v[10:11], -v[6:7], v[8:9], 1.0 +; GFX1150-NEXT: v_fma_f64 v[6:7], -v[8:9], v[6:7], v[2:3] +; GFX1150-NEXT: v_bfi_b32 v7, 0x7fffffff, v7, v3 +; GFX1150-NEXT: v_div_scale_f64 v[2:3], null, v[4:5], v[4:5], v[0:1] +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1) +; GFX1150-NEXT: v_rcp_f64_e32 v[8:9], v[2:3] +; GFX1150-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] +; GFX1150-NEXT: v_fma_f64 v[10:11], -v[2:3], v[8:9], 1.0 +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_fma_f64 v[8:9], v[8:9], v[10:11], v[8:9] ; GFX1150-NEXT: v_div_scale_f64 v[10:11], vcc_lo, v[0:1], v[4:5], v[0:1] -; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) ; GFX1150-NEXT: v_mul_f64 v[12:13], v[10:11], v[8:9] -; GFX1150-NEXT: v_fma_f64 v[6:7], -v[6:7], v[12:13], v[10:11] ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_div_fmas_f64 v[6:7], v[6:7], v[8:9], v[12:13] -; GFX1150-NEXT: v_div_fixup_f64 v[6:7], v[6:7], v[4:5], v[0:1] +; GFX1150-NEXT: v_fma_f64 v[2:3], -v[2:3], v[12:13], v[10:11] +; GFX1150-NEXT: v_div_fmas_f64 v[2:3], v[2:3], v[8:9], v[12:13] +; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX1150-NEXT: v_div_fixup_f64 v[2:3], v[2:3], v[4:5], v[0:1] +; GFX1150-NEXT: v_trunc_f64_e32 v[2:3], v[2:3] ; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) -; GFX1150-NEXT: v_trunc_f64_e32 v[6:7], v[6:7] -; GFX1150-NEXT: v_fma_f64 v[0:1], -v[6:7], v[4:5], v[0:1] -; GFX1150-NEXT: global_store_b128 v16, v[0:3], s[4:5] +; GFX1150-NEXT: v_fma_f64 v[4:5], -v[2:3], v[4:5], v[0:1] +; GFX1150-NEXT: v_bfi_b32 v5, 0x7fffffff, v5, v1 +; GFX1150-NEXT: global_store_b128 v16, v[4:7], s[4:5] ; GFX1150-NEXT: s_nop 0 ; GFX1150-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) ; GFX1150-NEXT: s_endpgm diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll index cadc23414dcac..5fad26ea16d13 100644 --- a/llvm/test/CodeGen/AMDGPU/wave32.ll +++ b/llvm/test/CodeGen/AMDGPU/wave32.ll @@ -2584,7 +2584,8 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) { ; GFX1032-NEXT: v_div_fmas_f32 v1, v1, v2, v3 ; GFX1032-NEXT: v_div_fixup_f32 v1, v1, s0, v0 ; GFX1032-NEXT: v_trunc_f32_e32 v1, v1 -; GFX1032-NEXT: v_fma_f32 v0, -v1, s0, v0 +; GFX1032-NEXT: v_fma_f32 v1, -v1, s0, v0 +; GFX1032-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0 ; GFX1032-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 ; GFX1032-NEXT: s_lshr_b32 s0, vcc_lo, 1 ; GFX1032-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0 @@ -2616,7 +2617,8 @@ define amdgpu_kernel void @fcmp64(float %n, float %s) { ; GFX1064-NEXT: v_div_fmas_f32 v1, v1, v2, v3 ; GFX1064-NEXT: v_div_fixup_f32 v1, v1, s2, v0 ; GFX1064-NEXT: v_trunc_f32_e32 v1, v1 -; GFX1064-NEXT: v_fma_f32 v0, -v1, s2, v0 +; GFX1064-NEXT: v_fma_f32 v1, -v1, s2, v0 +; GFX1064-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0 ; GFX1064-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0 ; GFX1064-NEXT: s_lshr_b64 s[0:1], vcc, 1 ; GFX1064-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0 @@ -2773,7 +2775,8 @@ define amdgpu_kernel void @fcmp32(float %n, float %s) { ; GFX1032-NEXT: v_div_fmas_f32 v1, v1, v2, v4 ; GFX1032-NEXT: v_div_fixup_f32 v1, v1, s0, v0 ; GFX1032-NEXT: v_trunc_f32_e32 v1, v1 -; GFX1032-NEXT: v_fma_f32 v0, -v1, s0, v0 +; GFX1032-NEXT: v_fma_f32 v1, -v1, s0, v0 +; GFX1032-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0 ; GFX1032-NEXT: v_cmp_eq_f32_e32 vcc_lo, 0, v0 ; GFX1032-NEXT: s_lshr_b32 s0, vcc_lo, 1 ; GFX1032-NEXT: v_cmp_nlg_f32_e32 vcc_lo, 0, v0 @@ -2805,7 +2808,8 @@ define amdgpu_kernel void @fcmp32(float %n, float %s) { ; GFX1064-NEXT: v_div_fmas_f32 v1, v1, v2, v4 ; GFX1064-NEXT: v_div_fixup_f32 v1, v1, s2, v0 ; GFX1064-NEXT: v_trunc_f32_e32 v1, v1 -; GFX1064-NEXT: v_fma_f32 v0, -v1, s2, v0 +; GFX1064-NEXT: v_fma_f32 v1, -v1, s2, v0 +; GFX1064-NEXT: v_bfi_b32 v0, 0x7fffffff, v1, v0 ; GFX1064-NEXT: v_cmp_eq_f32_e32 vcc, 0, v0 ; GFX1064-NEXT: s_lshr_b32 s0, vcc_lo, 1 ; GFX1064-NEXT: v_cmp_nlg_f32_e32 vcc, 0, v0