Revert "Redesign Straight-Line Strength Reduction (SLSR) (#162930)" (#169546)

This reverts commit f67409c.

cc @fiigii 
Including us, several separate groups are experiencing regressions with
this change. This is the smallest reproducer pasted by @akuegel :
#162930 (comment)

Author: lialan

llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp

@@ -541,9 +541,10 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    s_lshr_b32 s2, s0, 16
 ; GFX908-NEXT:    v_cvt_f32_f16_e32 v19, s2
 ; GFX908-NEXT:    s_lshl_b64 s[6:7], s[4:5], 5
-; GFX908-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX908-NEXT:    s_lshl_b64 s[14:15], s[10:11], 5
+; GFX908-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX908-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
+; GFX908-NEXT:    s_or_b32 s14, s14, 28
 ; GFX908-NEXT:    s_lshl_b64 s[16:17], s[8:9], 5
 ; GFX908-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
@@ -609,13 +610,13 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    ; => This Inner Loop Header: Depth=2
 ; GFX908-NEXT:    s_add_u32 s22, s20, s9
 ; GFX908-NEXT:    s_addc_u32 s23, s21, s13
-; GFX908-NEXT:    global_load_dword v21, v17, s[22:23] offset:16 glc
+; GFX908-NEXT:    global_load_dword v21, v17, s[22:23] offset:-12 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    global_load_dword v20, v17, s[22:23] offset:20 glc
+; GFX908-NEXT:    global_load_dword v20, v17, s[22:23] offset:-8 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    global_load_dword v12, v17, s[22:23] offset:24 glc
+; GFX908-NEXT:    global_load_dword v12, v17, s[22:23] offset:-4 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    global_load_dword v12, v17, s[22:23] offset:28 glc
+; GFX908-NEXT:    global_load_dword v12, v17, s[22:23] glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
 ; GFX908-NEXT:    ds_read_b64 v[12:13], v17
 ; GFX908-NEXT:    ds_read_b64 v[14:15], v0
@@ -709,6 +710,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    s_lshl_b64 s[6:7], s[4:5], 5
 ; GFX90A-NEXT:    s_lshl_b64 s[14:15], s[10:11], 5
 ; GFX90A-NEXT:    s_and_b64 s[0:1], exec, s[0:1]
+; GFX90A-NEXT:    s_or_b32 s14, s14, 28
 ; GFX90A-NEXT:    s_lshl_b64 s[16:17], s[8:9], 5
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    v_readfirstlane_b32 s2, v18
@@ -769,13 +771,13 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    ; => This Inner Loop Header: Depth=2
 ; GFX90A-NEXT:    s_add_u32 s22, s20, s9
 ; GFX90A-NEXT:    s_addc_u32 s23, s21, s13
-; GFX90A-NEXT:    global_load_dword v21, v19, s[22:23] offset:16 glc
+; GFX90A-NEXT:    global_load_dword v21, v19, s[22:23] offset:-12 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_load_dword v20, v19, s[22:23] offset:20 glc
+; GFX90A-NEXT:    global_load_dword v20, v19, s[22:23] offset:-8 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_load_dword v14, v19, s[22:23] offset:24 glc
+; GFX90A-NEXT:    global_load_dword v14, v19, s[22:23] offset:-4 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_load_dword v14, v19, s[22:23] offset:28 glc
+; GFX90A-NEXT:    global_load_dword v14, v19, s[22:23] glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    ds_read_b64 v[14:15], v19
 ; GFX90A-NEXT:    ds_read_b64 v[16:17], v0

llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll

@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=amdgcn -amdgpu-scalar-ir-passes=false < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck %s
 
 ; Test for a bug where DAGCombiner::ReassociateOps() was creating adds
 ; with offset in the first operand and base pointers in the second.

llvm/test/CodeGen/AMDGPU/dagcombine-reassociate-bug.ll

@@ -2396,7 +2396,7 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(ptr addrspace(1) %src1,
 ; GFX9-NODL-NEXT:    v_mul_u32_u24_e32 v4, v2, v1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    v_mad_u32_u24 v1, v2, v1, s0
-; GFX9-NODL-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX9-NODL-NEXT:    v_add3_u32 v1, v4, v1, v3
 ; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[6:7]
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2417,7 +2417,7 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(ptr addrspace(1) %src1,
 ; GFX9-DL-NEXT:    v_mul_u32_u24_e32 v4, v2, v1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    v_mad_u32_u24 v1, v2, v1, s0
-; GFX9-DL-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX9-DL-NEXT:    v_add3_u32 v1, v4, v1, v3
 ; GFX9-DL-NEXT:    global_store_dword v0, v1, s[6:7]
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2442,7 +2442,7 @@ define amdgpu_kernel void @udot2_MultipleUses_mul2(ptr addrspace(1) %src1,
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    v_mad_u32_u24 v0, v3, v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-DL-NEXT:    v_add3_u32 v0, v0, v2, v1
+; GFX10-DL-NEXT:    v_add3_u32 v0, v2, v0, v1
 ; GFX10-DL-NEXT:    global_store_dword v3, v0, s[6:7]
 ; GFX10-DL-NEXT:    s_endpgm
                                                    ptr addrspace(1) %src2,
@@ -2553,7 +2553,7 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(ptr addrspace(1) %src1,
 ; GFX9-NODL-NEXT:    v_mul_i32_i24_e32 v4, v2, v1
 ; GFX9-NODL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NODL-NEXT:    v_mad_i32_i24 v1, v2, v1, s0
-; GFX9-NODL-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX9-NODL-NEXT:    v_add3_u32 v1, v4, v1, v3
 ; GFX9-NODL-NEXT:    global_store_dword v0, v1, s[6:7]
 ; GFX9-NODL-NEXT:    s_endpgm
 ;
@@ -2574,7 +2574,7 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(ptr addrspace(1) %src1,
 ; GFX9-DL-NEXT:    v_mul_i32_i24_e32 v4, v2, v1
 ; GFX9-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-DL-NEXT:    v_mad_i32_i24 v1, v2, v1, s0
-; GFX9-DL-NEXT:    v_add3_u32 v1, v1, v4, v3
+; GFX9-DL-NEXT:    v_add3_u32 v1, v4, v1, v3
 ; GFX9-DL-NEXT:    global_store_dword v0, v1, s[6:7]
 ; GFX9-DL-NEXT:    s_endpgm
 ;
@@ -2599,7 +2599,7 @@ define amdgpu_kernel void @idot2_MultipleUses_mul2(ptr addrspace(1) %src1,
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    v_mad_i32_i24 v0, v3, v0, s0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-DL-NEXT:    v_add3_u32 v0, v0, v2, v1
+; GFX10-DL-NEXT:    v_add3_u32 v0, v2, v0, v1
 ; GFX10-DL-NEXT:    global_store_dword v3, v0, s[6:7]
 ; GFX10-DL-NEXT:    s_endpgm
                                                    ptr addrspace(1) %src2,

llvm/test/CodeGen/AMDGPU/idot2.ll

@@ -3268,19 +3268,19 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX7-NEXT:    buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
 ; GFX7-NEXT:    s_mov_b32 s2, -1
 ; GFX7-NEXT:    s_waitcnt vmcnt(1)
-; GFX7-NEXT:    v_bfe_i32 v3, v2, 8, 8
 ; GFX7-NEXT:    v_bfe_i32 v1, v2, 0, 8
+; GFX7-NEXT:    v_bfe_i32 v3, v2, 8, 8
+; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff, v1
 ; GFX7-NEXT:    s_waitcnt vmcnt(0)
+; GFX7-NEXT:    v_and_b32_e32 v5, 0xff, v0
+; GFX7-NEXT:    v_bfe_i32 v4, v2, 16, 8
 ; GFX7-NEXT:    v_bfe_u32 v6, v0, 8, 8
 ; GFX7-NEXT:    v_and_b32_e32 v3, 0xffff, v3
-; GFX7-NEXT:    v_bfe_i32 v4, v2, 16, 8
-; GFX7-NEXT:    v_and_b32_e32 v1, 0xffff, v1
-; GFX7-NEXT:    v_and_b32_e32 v5, 0xff, v0
-; GFX7-NEXT:    v_mul_u32_u24_e32 v3, v6, v3
+; GFX7-NEXT:    v_mul_u32_u24_e32 v1, v1, v5
 ; GFX7-NEXT:    v_ashrrev_i32_e32 v2, 24, v2
 ; GFX7-NEXT:    v_bfe_u32 v7, v0, 16, 8
 ; GFX7-NEXT:    v_and_b32_e32 v4, 0xffff, v4
-; GFX7-NEXT:    v_mad_u32_u24 v1, v1, v5, v3
+; GFX7-NEXT:    v_mad_u32_u24 v1, v6, v3, v1
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
 ; GFX7-NEXT:    v_mad_u32_u24 v1, v7, v4, v1
 ; GFX7-NEXT:    v_and_b32_e32 v2, 0xffff, v2
@@ -3307,18 +3307,18 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_waitcnt vmcnt(1)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v3
+; GFX8-NEXT:    v_lshrrev_b32_e32 v7, 8, v3
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GFX8-NEXT:    v_bfe_i32 v6, v3, 0, 8
-; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
+; GFX8-NEXT:    v_bfe_i32 v7, v7, 0, 8
 ; GFX8-NEXT:    v_bfe_i32 v5, v5, 0, 8
-; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_lshrrev_b32_e32 v9, 8, v2
-; GFX8-NEXT:    v_and_b32_e32 v7, 0xff, v2
-; GFX8-NEXT:    v_mul_lo_u16_sdwa v8, v9, sext(v8) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_lshrrev_b32_e32 v8, 8, v2
+; GFX8-NEXT:    v_mul_lo_u16_sdwa v6, sext(v3), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX8-NEXT:    v_and_b32_e32 v8, 0xff, v8
 ; GFX8-NEXT:    v_and_b32_sdwa v4, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX8-NEXT:    v_mad_u16 v6, v6, v7, v8
+; GFX8-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
+; GFX8-NEXT:    v_mad_u16 v6, v8, v7, v6
+; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 8
 ; GFX8-NEXT:    v_mad_u16 v4, v4, v5, v6
 ; GFX8-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
 ; GFX8-NEXT:    v_mad_u16 v2, v3, v2, v4
@@ -3337,19 +3337,19 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX9-NODL-NEXT:    s_movk_i32 s0, 0xff
 ; GFX9-NODL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
 ; GFX9-NODL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v7, 8, v2
+; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
 ; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX9-NODL-NEXT:    v_bfe_i32 v4, v1, 0, 8
-; GFX9-NODL-NEXT:    v_and_b32_e32 v5, 0xff, v2
-; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v6, v7, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-NODL-NEXT:    v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-NODL-NEXT:    v_mul_lo_u16_sdwa v4, sext(v1), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-NODL-NEXT:    v_bfe_i32 v5, v5, 0, 8
+; GFX9-NODL-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GFX9-NODL-NEXT:    v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
 ; GFX9-NODL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX9-NODL-NEXT:    v_mad_legacy_u16 v4, v4, v5, v6
+; GFX9-NODL-NEXT:    v_mad_legacy_u16 v4, v6, v5, v4
 ; GFX9-NODL-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX9-NODL-NEXT:    v_mad_legacy_u16 v3, v8, v3, v4
+; GFX9-NODL-NEXT:    v_mad_legacy_u16 v3, v7, v3, v4
 ; GFX9-NODL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
 ; GFX9-NODL-NEXT:    v_mad_legacy_u16 v1, v1, v2, v3
 ; GFX9-NODL-NEXT:    v_bfe_i32 v1, v1, 0, 16
@@ -3367,19 +3367,19 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX9-DL-NEXT:    s_movk_i32 s0, 0xff
 ; GFX9-DL-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v6, 8, v1
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v1
 ; GFX9-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v7, 8, v2
+; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v6, 8, v2
 ; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v3, 16, v1
-; GFX9-DL-NEXT:    v_bfe_i32 v4, v1, 0, 8
-; GFX9-DL-NEXT:    v_and_b32_e32 v5, 0xff, v2
-; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v6, v7, sext(v6) dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
-; GFX9-DL-NEXT:    v_and_b32_sdwa v8, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX9-DL-NEXT:    v_mul_lo_u16_sdwa v4, sext(v1), v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_0
+; GFX9-DL-NEXT:    v_bfe_i32 v5, v5, 0, 8
+; GFX9-DL-NEXT:    v_and_b32_e32 v6, 0xff, v6
+; GFX9-DL-NEXT:    v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
 ; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
 ; GFX9-DL-NEXT:    v_bfe_i32 v3, v3, 0, 8
-; GFX9-DL-NEXT:    v_mad_legacy_u16 v4, v4, v5, v6
+; GFX9-DL-NEXT:    v_mad_legacy_u16 v4, v6, v5, v4
 ; GFX9-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX9-DL-NEXT:    v_mad_legacy_u16 v3, v8, v3, v4
+; GFX9-DL-NEXT:    v_mad_legacy_u16 v3, v7, v3, v4
 ; GFX9-DL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
 ; GFX9-DL-NEXT:    v_mad_legacy_u16 v1, v1, v2, v3
 ; GFX9-DL-NEXT:    v_bfe_i32 v1, v1, 0, 16
@@ -3392,28 +3392,28 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX10-DL-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x24
 ; GFX10-DL-NEXT:    s_load_dwordx2 s[6:7], s[4:5], 0x34
 ; GFX10-DL-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
-; GFX10-DL-NEXT:    v_mov_b32_e32 v4, 0xff
+; GFX10-DL-NEXT:    v_mov_b32_e32 v6, 0xff
 ; GFX10-DL-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-DL-NEXT:    s_clause 0x1
 ; GFX10-DL-NEXT:    global_load_dword v1, v0, s[0:1]
 ; GFX10-DL-NEXT:    global_load_dword v2, v0, s[2:3]
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v0, 8, v1
+; GFX10-DL-NEXT:    v_bfe_i32 v0, v1, 0, 8
 ; GFX10-DL-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v3, 8, v2
-; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 16, v1
-; GFX10-DL-NEXT:    v_bfe_i32 v6, v1, 0, 8
-; GFX10-DL-NEXT:    v_and_b32_e32 v7, 0xff, v2
-; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 8
-; GFX10-DL-NEXT:    v_and_b32_e32 v3, 0xff, v3
+; GFX10-DL-NEXT:    v_and_b32_e32 v3, 0xff, v2
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v5, 8, v2
+; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
 ; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
-; GFX10-DL-NEXT:    v_mul_lo_u16 v0, v3, v0
-; GFX10-DL-NEXT:    v_and_b32_sdwa v3, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
-; GFX10-DL-NEXT:    v_bfe_i32 v4, v5, 0, 8
+; GFX10-DL-NEXT:    v_mul_lo_u16 v0, v0, v3
+; GFX10-DL-NEXT:    v_bfe_i32 v3, v4, 0, 8
+; GFX10-DL-NEXT:    v_and_b32_e32 v4, 0xff, v5
+; GFX10-DL-NEXT:    v_and_b32_sdwa v5, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
+; GFX10-DL-NEXT:    v_bfe_i32 v6, v7, 0, 8
 ; GFX10-DL-NEXT:    v_bfe_i32 v1, v1, 0, 8
 ; GFX10-DL-NEXT:    v_lshrrev_b32_e32 v2, 24, v2
-; GFX10-DL-NEXT:    v_mad_u16 v0, v6, v7, v0
-; GFX10-DL-NEXT:    v_mad_u16 v0, v3, v4, v0
+; GFX10-DL-NEXT:    v_mad_u16 v0, v4, v3, v0
+; GFX10-DL-NEXT:    v_mad_u16 v0, v5, v6, v0
 ; GFX10-DL-NEXT:    v_mad_u16 v0, v1, v2, v0
 ; GFX10-DL-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX10-DL-NEXT:    v_bfe_i32 v0, v0, 0, 16
@@ -3429,34 +3429,32 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX11-DL-TRUE16-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-DL-TRUE16-NEXT:    s_clause 0x1
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v3, v0, s[0:1]
-; GFX11-DL-TRUE16-NEXT:    global_load_b32 v4, v0, s[2:3]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v2, v0, s[0:1]
+; GFX11-DL-TRUE16-NEXT:    global_load_b32 v3, v0, s[2:3]
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v2
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v1, v2, 0, 8
 ; GFX11-DL-TRUE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v1, 8, v4
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v3, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v3.h
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v2, v0, 0, 8
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v1.l
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v4.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.l
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.l, 0xff, v3.l
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v3
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v6.l, v2.h
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.h, 0xff, v3.h
+; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_and_b16 v0.h, 0xff, v5.l
 ; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v6, 24, v3
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-DL-TRUE16-NEXT:    v_mul_lo_u16 v0.l, v0.l, v1.l
-; GFX11-DL-TRUE16-NEXT:    v_and_b16 v1.l, 0xff, v4.h
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v3.l, v5.l
-; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 24, v4
-; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v2.l, v0.h, v0.l
-; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v2, v6, 0, 8
-; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.l, v3.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v4.l
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v4, 24, v2
+; GFX11-DL-TRUE16-NEXT:    v_lshrrev_b32_e32 v3, 24, v3
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v2.l, v5.l
+; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v0.h, v1.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v4, v4, 0, 8
+; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.h, v2.l, v0.l
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v2.l
-; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.l, v4.l, v0.l
+; GFX11-DL-TRUE16-NEXT:    v_mov_b16_e32 v1.l, v4.l
+; GFX11-DL-TRUE16-NEXT:    v_mad_u16 v0.l, v1.l, v3.l, v0.l
 ; GFX11-DL-TRUE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX11-DL-TRUE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)
 ; GFX11-DL-TRUE16-NEXT:    v_bfe_i32 v0, v0, 0, 16
@@ -3475,25 +3473,24 @@ define amdgpu_kernel void @idot4_nonstandard_signed(ptr addrspace(1) %src1,
 ; GFX11-DL-FAKE16-NEXT:    global_load_b32 v1, v0, s[0:1]
 ; GFX11-DL-FAKE16-NEXT:    global_load_b32 v0, v0, s[2:3]
 ; GFX11-DL-FAKE16-NEXT:    s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v2, 8, v1
+; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v2, v1, 0, 8
 ; GFX11-DL-FAKE16-NEXT:    s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v3, 8, v0
-; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 16, v1
-; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
-; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v6, v1, 0, 8
-; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v2, v2, 0, 8
-; GFX11-DL-FAKE16-NEXT:    v_and_b32_e32 v3, 0xff, v3
-; GFX11-DL-FAKE16-NEXT:    v_and_b32_e32 v7, 0xff, v0
-; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
-; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
-; GFX11-DL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-DL-FAKE16-NEXT:    v_mul_lo_u16 v2, v3, v2
+; GFX11-DL-FAKE16-NEXT:    v_and_b32_e32 v3, 0xff, v0
+; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v5, 8, v0
+; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v6, 16, v1
+; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v7, 16, v0
+; GFX11-DL-FAKE16-NEXT:    v_mul_lo_u16 v2, v2, v3
 ; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v3, v4, 0, 8
 ; GFX11-DL-FAKE16-NEXT:    v_and_b32_e32 v4, 0xff, v5
-; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
-; GFX11-DL-FAKE16-NEXT:    v_mad_u16 v2, v6, v7, v2
-; GFX11-DL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v1, 24, v1
+; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v5, v6, 0, 8
+; GFX11-DL-FAKE16-NEXT:    v_and_b32_e32 v6, 0xff, v7
+; GFX11-DL-FAKE16-NEXT:    v_lshrrev_b32_e32 v0, 24, v0
 ; GFX11-DL-FAKE16-NEXT:    v_mad_u16 v2, v4, v3, v2
+; GFX11-DL-FAKE16-NEXT:    v_bfe_i32 v1, v1, 0, 8
+; GFX11-DL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT:    v_mad_u16 v2, v6, v5, v2
 ; GFX11-DL-FAKE16-NEXT:    v_mad_u16 v0, v1, v0, v2
 ; GFX11-DL-FAKE16-NEXT:    v_mov_b32_e32 v1, 0
 ; GFX11-DL-FAKE16-NEXT:    s_delay_alu instid0(VALU_DEP_2)