From 051846c55cc0927a5abacc235583e755cd92bab5 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Fri, 12 Apr 2024 14:15:32 +0100 Subject: [PATCH 01/11] [AMDGPU] Use LSH for lowering ctlz_zero_undef.i8/i16 --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 19 +++-- llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 77 +++++++++---------- 2 files changed, 52 insertions(+), 44 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index d35a022ad6806..e06837a7ab517 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -3117,20 +3117,29 @@ static bool isCttzOpc(unsigned Opc) { SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op, SelectionDAG &DAG) const { auto SL = SDLoc(Op); + auto Opc = Op.getOpcode(); auto Arg = Op.getOperand(0u); auto ResultVT = Op.getValueType(); if (ResultVT != MVT::i8 && ResultVT != MVT::i16) return {}; - assert(isCtlzOpc(Op.getOpcode())); + assert(isCtlzOpc(Opc)); assert(ResultVT == Arg.getValueType()); - auto const LeadingZeroes = 32u - ResultVT.getFixedSizeInBits(); - auto SubVal = DAG.getConstant(LeadingZeroes, SL, MVT::i32); + auto const NumBits = ResultVT.getFixedSizeInBits(); + auto NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32); auto NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); - NewOp = DAG.getNode(Op.getOpcode(), SL, MVT::i32, NewOp); - NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, SubVal); + + if (Opc == ISD::CTLZ_ZERO_UNDEF) { + NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits); + NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); + } + else { + NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); + NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits); + } + return DAG.getNode(ISD::TRUNCATE, SL, ResultVT, NewOp); } diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll index 54adde38d6d22..143ff5d88894b 100644 --- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll +++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll @@ -322,9 +322,8 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_and_b32 s2, s2, 0xff -; SI-NEXT: s_flbit_i32_b32 s2, s2 -; SI-NEXT: s_sub_i32 s4, s2, 24 +; SI-NEXT: s_lshl_b32 s2, s2, 24 +; SI-NEXT: s_flbit_i32_b32 s4, s2 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: v_mov_b32_e32 v0, s4 ; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0 @@ -335,9 +334,8 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; VI-NEXT: s_load_dword s2, s[0:1], 0x2c ; VI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24 ; VI-NEXT: s_waitcnt lgkmcnt(0) -; VI-NEXT: s_and_b32 s2, s2, 0xff +; VI-NEXT: s_lshl_b32 s2, s2, 24 ; VI-NEXT: s_flbit_i32_b32 s2, s2 -; VI-NEXT: s_sub_i32 s2, s2, 24 ; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: v_mov_b32_e32 v2, s2 @@ -357,13 +355,13 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, 0.0, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: FFBH_UINT T0.W, T0.X, +; EG-NEXT: LSHL * T0.W, T0.X, literal.x, +; EG-NEXT: 24(3.363116e-44), 0(0.000000e+00) +; EG-NEXT: FFBH_UINT T0.W, PV.W, ; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) -; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x, -; EG-NEXT: -24(nan), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, -; EG-NEXT: LSHL * T1.W, T1.W, literal.y, +; EG-NEXT: LSHL * T1.W, PS, literal.y, ; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45) ; EG-NEXT: LSHL T0.X, PV.W, PS, ; EG-NEXT: LSHL * T0.W, literal.x, PS, @@ -399,9 +397,8 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; SI-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x9 ; SI-NEXT: s_mov_b32 s3, 0xf000 ; SI-NEXT: s_waitcnt lgkmcnt(0) -; SI-NEXT: s_and_b32 s2, s2, 0xffff -; SI-NEXT: s_flbit_i32_b32 s2, s2 -; SI-NEXT: s_add_i32 s4, s2, -16 +; SI-NEXT: s_lshl_b32 s2, s2, 16 +; SI-NEXT: s_flbit_i32_b32 s4, s2 ; SI-NEXT: s_mov_b32 s2, -1 ; SI-NEXT: v_mov_b32_e32 v0, s4 ; SI-NEXT: buffer_store_short v0, off, s[0:3], 0 @@ -434,13 +431,13 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, 0.0, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: FFBH_UINT T0.W, T0.X, +; EG-NEXT: LSHL * T0.W, T0.X, literal.x, +; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) +; EG-NEXT: FFBH_UINT T0.W, PV.W, ; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) -; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x, -; EG-NEXT: -16(nan), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, -; EG-NEXT: LSHL * T1.W, T1.W, literal.y, +; EG-NEXT: LSHL * T1.W, PS, literal.y, ; EG-NEXT: 65535(9.183409e-41), 3(4.203895e-45) ; EG-NEXT: LSHL T0.X, PV.W, PS, ; EG-NEXT: LSHL * T0.W, literal.x, PS, @@ -598,8 +595,8 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; SI-NEXT: s_mov_b32 s4, s0 ; SI-NEXT: s_mov_b32 s5, s1 ; SI-NEXT: s_waitcnt vmcnt(0) -; SI-NEXT: v_ffbh_u32_e32 v1, v0 -; SI-NEXT: v_subrev_i32_e32 v1, vcc, 24, v1 +; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v0 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 ; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 ; SI-NEXT: v_cndmask_b32_e32 v0, 32, v1, vcc ; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0 @@ -613,8 +610,8 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; VI-NEXT: v_mov_b32_e32 v1, s3 ; VI-NEXT: flat_load_ubyte v0, v[0:1] ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; VI-NEXT: v_subrev_u32_e32 v1, vcc, 24, v1 +; VI-NEXT: v_lshlrev_b32_e32 v1, 24, v0 +; VI-NEXT: v_ffbh_u32_e32 v1, v1 ; VI-NEXT: v_cmp_ne_u16_e32 vcc, 0, v0 ; VI-NEXT: v_cndmask_b32_e32 v2, 32, v1, vcc ; VI-NEXT: v_mov_b32_e32 v0, s0 @@ -626,7 +623,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; EG: ; %bb.0: ; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] ; EG-NEXT: TEX 0 @6 -; EG-NEXT: ALU 15, @9, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 16, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X ; EG-NEXT: CF_END ; EG-NEXT: PAD @@ -635,10 +632,11 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: FFBH_UINT * T0.W, T0.X, -; EG-NEXT: ADD_INT T0.W, PV.W, literal.x, -; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y, -; EG-NEXT: -24(nan), 3(4.203895e-45) +; EG-NEXT: LSHL * T0.W, T0.X, literal.x, +; EG-NEXT: 24(3.363116e-44), 0(0.000000e+00) +; EG-NEXT: FFBH_UINT T0.W, PV.W, +; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, +; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) ; EG-NEXT: CNDE_INT * T0.W, T0.X, literal.x, PV.W, ; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, @@ -693,8 +691,8 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; SI-NEXT: v_lshlrev_b32_e32 v0, 8, v0 ; SI-NEXT: s_waitcnt vmcnt(0) ; SI-NEXT: v_or_b32_e32 v0, v0, v1 -; SI-NEXT: v_ffbh_u32_e32 v1, v0 -; SI-NEXT: v_add_i32_e32 v1, vcc, -16, v1 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v0 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 ; SI-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0 ; SI-NEXT: v_cndmask_b32_e32 v0, 32, v1, vcc ; SI-NEXT: buffer_store_short v0, off, s[4:7], 0 @@ -729,7 +727,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; EG: ; %bb.0: ; EG-NEXT: ALU 0, @8, KC0[CB0:0-32], KC1[] ; EG-NEXT: TEX 0 @6 -; EG-NEXT: ALU 15, @9, KC0[CB0:0-32], KC1[] +; EG-NEXT: ALU 16, @9, KC0[CB0:0-32], KC1[] ; EG-NEXT: MEM_RAT MSKOR T0.XW, T1.X ; EG-NEXT: CF_END ; EG-NEXT: PAD @@ -738,10 +736,11 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: MOV * T0.X, KC0[2].Z, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: FFBH_UINT * T0.W, T0.X, -; EG-NEXT: ADD_INT T0.W, PV.W, literal.x, -; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.y, -; EG-NEXT: -16(nan), 3(4.203895e-45) +; EG-NEXT: LSHL * T0.W, T0.X, literal.x, +; EG-NEXT: 16(2.242078e-44), 0(0.000000e+00) +; EG-NEXT: FFBH_UINT T0.W, PV.W, +; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, +; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) ; EG-NEXT: CNDE_INT * T0.W, T0.X, literal.x, PV.W, ; EG-NEXT: 32(4.484155e-44), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, @@ -1110,8 +1109,8 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, p ; SI-NEXT: s_mov_b32 s4, s0 ; SI-NEXT: s_mov_b32 s5, s1 ; SI-NEXT: s_waitcnt vmcnt(0) +; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0 ; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_subrev_i32_e32 v0, vcc, 24, v0 ; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0 ; SI-NEXT: s_endpgm ; @@ -1124,8 +1123,8 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, p ; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc ; VI-NEXT: flat_load_ubyte v0, v[0:1] ; VI-NEXT: s_waitcnt vmcnt(0) -; VI-NEXT: v_ffbh_u32_e32 v0, v0 -; VI-NEXT: v_subrev_u32_e32 v2, vcc, 24, v0 +; VI-NEXT: v_lshlrev_b32_e32 v0, 24, v0 +; VI-NEXT: v_ffbh_u32_e32 v2, v0 ; VI-NEXT: v_mov_b32_e32 v0, s0 ; VI-NEXT: v_mov_b32_e32 v1, s1 ; VI-NEXT: flat_store_byte v[0:1], v2 @@ -1144,13 +1143,13 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, p ; EG-NEXT: ALU clause starting at 8: ; EG-NEXT: ADD_INT * T0.X, KC0[2].Z, T0.X, ; EG-NEXT: ALU clause starting at 9: -; EG-NEXT: FFBH_UINT T0.W, T0.X, +; EG-NEXT: LSHL * T0.W, T0.X, literal.x, +; EG-NEXT: 24(3.363116e-44), 0(0.000000e+00) +; EG-NEXT: FFBH_UINT T0.W, PV.W, ; EG-NEXT: AND_INT * T1.W, KC0[2].Y, literal.x, ; EG-NEXT: 3(4.203895e-45), 0(0.000000e+00) -; EG-NEXT: ADD_INT * T0.W, PV.W, literal.x, -; EG-NEXT: -24(nan), 0(0.000000e+00) ; EG-NEXT: AND_INT T0.W, PV.W, literal.x, -; EG-NEXT: LSHL * T1.W, T1.W, literal.y, +; EG-NEXT: LSHL * T1.W, PS, literal.y, ; EG-NEXT: 255(3.573311e-43), 3(4.203895e-45) ; EG-NEXT: LSHL T0.X, PV.W, PS, ; EG-NEXT: LSHL * T0.W, literal.x, PS, From 6a8e358a0815e43b897f1eca33a6417c3443b520 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Fri, 12 Apr 2024 14:43:42 +0100 Subject: [PATCH 02/11] Formatting changes. --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index e06837a7ab517..eff5734b07b81 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -3134,8 +3134,7 @@ SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op, if (Opc == ISD::CTLZ_ZERO_UNDEF) { NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits); NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); - } - else { + } else { NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits); } From 5d41bdbdb65099d5d651a5261d002866efe1f1f3 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Mon, 22 Apr 2024 22:51:34 +0100 Subject: [PATCH 03/11] Add optimisations to GISel and address comments. --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 4 +- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 50 ++++++++++++++++++- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 2 + .../GlobalISel/legalize-ctlz-zero-undef.mir | 33 ++++++------ llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 24 ++++----- 5 files changed, 77 insertions(+), 36 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index eff5734b07b81..d01d0483af071 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -3129,12 +3129,14 @@ SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op, auto const NumBits = ResultVT.getFixedSizeInBits(); auto NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32); - auto NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); + auto NewOp = SDValue(); if (Opc == ISD::CTLZ_ZERO_UNDEF) { + NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg); NewOp = DAG.getNode(ISD::SHL, SL, MVT::i32, NewOp, NumExtBits); NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); } else { + NewOp = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Arg); NewOp = DAG.getNode(Opc, SL, MVT::i32, NewOp); NewOp = DAG.getNode(ISD::SUB, SL, MVT::i32, NewOp, NumExtBits); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index bd7bf78c4c0bd..c241522896084 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -1270,7 +1270,30 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .custom(); // The 64-bit versions produce 32-bit results, but only on the SALU. - getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF}) +// getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF}) +// .legalFor({{S32, S32}, {S32, S64}}) +// .clampScalar(0, S32, S32) +// .clampScalar(1, S32, S64) +// .scalarize(0) +// .widenScalarToNextPow2(0, 32) +// .widenScalarToNextPow2(1, 32); + + getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF) + .legalFor({{S32, S32}, {S32, S64}}) + .customFor({{S32, S8}, {S32, S16}}) + .clampScalar(0, S32, S32) + .clampScalar(1, S32, S64) + .scalarize(0) + .widenScalarToNextPow2(0, 32) + .widenScalarToNextPow2(1, 32); +// .custom(); + +// .legalFor({S32}) +// .customFor({S64}) +// .clampScalar(0, S32, S64) +// .scalarize(0); + + getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF) .legalFor({{S32, S32}, {S32, S64}}) .clampScalar(0, S32, S32) .clampScalar(1, S32, S64) @@ -2128,6 +2151,8 @@ bool AMDGPULegalizerInfo::legalizeCustom( case TargetOpcode::G_CTLZ: case TargetOpcode::G_CTTZ: return legalizeCTLZ_CTTZ(MI, MRI, B); + case TargetOpcode::G_CTLZ_ZERO_UNDEF: + return legalizeCTLZ_ZERO_UNDEF(MI, MRI, B); case TargetOpcode::G_INTRINSIC_FPTRUNC_ROUND: return legalizeFPTruncRound(MI, B); case TargetOpcode::G_STACKSAVE: @@ -4145,6 +4170,29 @@ bool AMDGPULegalizerInfo::legalizeCTLZ_CTTZ(MachineInstr &MI, return true; } +bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { + auto Dst = MI.getOperand(0).getReg(); + auto Src = MI.getOperand(1).getReg(); + auto DstTy = MRI.getType(Dst); + auto SrcTy = MRI.getType(Src); + auto NumBits = SrcTy.getSizeInBits(); + + assert(NumBits < 32u); + + auto ShiftAmt = B.buildConstant(S32, 32u - NumBits); + Src = B.buildAnyExt(S32, {Src}).getReg(0u); + Src = B.buildLShr(S32, {Src}, ShiftAmt).getReg(0u); + B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Src}); + MI.eraseFromParent(); + return true; + + // LocalAccum = B.buildZExt(S32, CarryIn[0]).getReg(0); + // auto ShiftAmt = B.buildConstant(S32, Shift); + // AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0); +} + // Check that this is a G_XOR x, -1 static bool isNot(const MachineRegisterInfo &MRI, const MachineInstr &MI) { if (MI.getOpcode() != TargetOpcode::G_XOR) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index e5ba84a74a0f8..4b1d821dadc21 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -108,6 +108,8 @@ class AMDGPULegalizerInfo final : public LegalizerInfo { bool legalizeMul(LegalizerHelper &Helper, MachineInstr &MI) const; bool legalizeCTLZ_CTTZ(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; + bool legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool loadInputValue(Register DstReg, MachineIRBuilder &B, const ArgDescriptor *Arg, diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir index fed277d7d10d0..32af5e1975a6c 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir @@ -81,14 +81,12 @@ body: | ; CHECK: liveins: $vgpr0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535 - ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]] - ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[AND]](s32) - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 16 - ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[CTLZ_ZERO_UNDEF]], [[C1]] - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) - ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]] - ; CHECK-NEXT: $vgpr0 = COPY [[AND1]](s32) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16 + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32) + ; CHECK-NEXT: [[AMDGPU_FFBH_U32:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[LSHR]](s32) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535 + ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32]], [[C1]] + ; CHECK-NEXT: $vgpr0 = COPY [[AND]](s32) %0:_(s32) = COPY $vgpr0 %1:_(s16) = G_TRUNC %0 %2:_(s16) = G_CTLZ_ZERO_UNDEF %1 @@ -149,18 +147,15 @@ body: | ; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(s32) = G_BITCAST [[COPY]](<2 x s16>) ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 16 ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32) + ; CHECK-NEXT: [[LSHR1:%[0-9]+]]:_(s32) = G_LSHR [[BITCAST]], [[C]](s32) + ; CHECK-NEXT: [[AMDGPU_FFBH_U32:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[LSHR1]](s32) + ; CHECK-NEXT: [[LSHR2:%[0-9]+]]:_(s32) = G_LSHR [[LSHR]], [[C]](s32) + ; CHECK-NEXT: [[AMDGPU_FFBH_U321:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[LSHR2]](s32) ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 65535 - ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[BITCAST]], [[C1]] - ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[AND]](s32) - ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[CTLZ_ZERO_UNDEF]], [[C]] - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) - ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF1:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[LSHR]](s32) - ; CHECK-NEXT: [[SUB1:%[0-9]+]]:_(s32) = G_SUB [[CTLZ_ZERO_UNDEF1]], [[C]] - ; CHECK-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY [[SUB1]](s32) - ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C1]] - ; CHECK-NEXT: [[AND2:%[0-9]+]]:_(s32) = G_AND [[COPY2]], [[C1]] - ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND2]], [[C]](s32) - ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND1]], [[SHL]] + ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U32]], [[C1]] + ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[AMDGPU_FFBH_U321]], [[C1]] + ; CHECK-NEXT: [[SHL:%[0-9]+]]:_(s32) = G_SHL [[AND1]], [[C]](s32) + ; CHECK-NEXT: [[OR:%[0-9]+]]:_(s32) = G_OR [[AND]], [[SHL]] ; CHECK-NEXT: [[BITCAST1:%[0-9]+]]:_(<2 x s16>) = G_BITCAST [[OR]](s32) ; CHECK-NEXT: $vgpr0 = COPY [[BITCAST1]](<2 x s16>) %0:_(<2 x s16>) = COPY $vgpr0 diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll index 143ff5d88894b..9204eb5fdf8ca 100644 --- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll +++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll @@ -377,9 +377,8 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0xff +; GFX9-GISEL-NEXT: s_lshr_b32 s0, s4, 24 ; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0 -; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 24 ; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[2:3] ; GFX9-GISEL-NEXT: s_endpgm @@ -453,9 +452,8 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 ; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0xffff +; GFX9-GISEL-NEXT: s_lshr_b32 s0, s4, 16 ; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0 -; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 16 ; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-GISEL-NEXT: global_store_short v1, v0, s[2:3] ; GFX9-GISEL-NEXT: s_endpgm @@ -657,8 +655,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_with_select(ptr addrspace(1) noa ; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-GISEL-NEXT: global_load_ubyte v1, v0, s[2:3] ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 24, v2 +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v1 ; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xff, v2 ; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 ; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc @@ -763,8 +760,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i16_with_select(ptr addrspace(1) no ; GFX9-GISEL-NEXT: global_load_ubyte v2, v0, s[2:3] offset:1 ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) ; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v2, 8, v1 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v2, v1 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 16, v2 +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v1 ; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2 ; GFX9-GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 ; GFX9-GISEL-NEXT: v_cndmask_b32_e32 v1, 32, v2, vcc @@ -1171,8 +1167,7 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8(ptr addrspace(1) noalias %out, p ; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off ; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0 ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 24, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 ; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1] ; GFX9-GISEL-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() @@ -1708,12 +1703,11 @@ define amdgpu_kernel void @v_ctlz_zero_undef_i8_sel_eq_neg1(ptr addrspace(1) noa ; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v1, v0 ; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v2, v3, vcc ; GFX9-GISEL-NEXT: global_load_ubyte v0, v[0:1], off -; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 24, v1 -; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0 -; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v1, -1, vcc ; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0 +; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v0 +; GFX9-GISEL-NEXT: v_cmp_eq_u32_sdwa s[2:3], v0, v1 +; GFX9-GISEL-NEXT: v_cndmask_b32_e64 v0, v2, -1, s[2:3] ; GFX9-GISEL-NEXT: global_store_byte v1, v0, s[0:1] ; GFX9-GISEL-NEXT: s_endpgm %tid = call i32 @llvm.amdgcn.workitem.id.x() From 0057bf25b4550ef3e6bcd3c943c825003d3188bb Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Tue, 23 Apr 2024 00:22:45 +0100 Subject: [PATCH 04/11] Remove commented code. --- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index c241522896084..b9f20d046e9b5 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -1270,14 +1270,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .custom(); // The 64-bit versions produce 32-bit results, but only on the SALU. -// getActionDefinitionsBuilder({G_CTLZ_ZERO_UNDEF, G_CTTZ_ZERO_UNDEF}) -// .legalFor({{S32, S32}, {S32, S64}}) -// .clampScalar(0, S32, S32) -// .clampScalar(1, S32, S64) -// .scalarize(0) -// .widenScalarToNextPow2(0, 32) -// .widenScalarToNextPow2(1, 32); - getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF) .legalFor({{S32, S32}, {S32, S64}}) .customFor({{S32, S8}, {S32, S16}}) @@ -1286,12 +1278,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .scalarize(0) .widenScalarToNextPow2(0, 32) .widenScalarToNextPow2(1, 32); -// .custom(); - -// .legalFor({S32}) -// .customFor({S64}) -// .clampScalar(0, S32, S64) -// .scalarize(0); getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF) .legalFor({{S32, S32}, {S32, S64}}) @@ -4187,10 +4173,6 @@ bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Src}); MI.eraseFromParent(); return true; - - // LocalAccum = B.buildZExt(S32, CarryIn[0]).getReg(0); - // auto ShiftAmt = B.buildConstant(S32, Shift); - // AndMaskSrc = B.buildLShr(S32, LiveIn, ShiftAmt).getReg(0); } // Check that this is a G_XOR x, -1 From 8b67341b0e36ace47fb23819bd823791aa6fb248 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Tue, 23 Apr 2024 00:43:07 +0100 Subject: [PATCH 05/11] Clang formatting. --- .../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index b9f20d046e9b5..20e5e42ca1f9a 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -1271,21 +1271,21 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // The 64-bit versions produce 32-bit results, but only on the SALU. getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF) - .legalFor({{S32, S32}, {S32, S64}}) - .customFor({{S32, S8}, {S32, S16}}) - .clampScalar(0, S32, S32) - .clampScalar(1, S32, S64) - .scalarize(0) - .widenScalarToNextPow2(0, 32) - .widenScalarToNextPow2(1, 32); + .legalFor({{S32, S32}, {S32, S64}}) + .customFor({{S32, S8}, {S32, S16}}) + .clampScalar(0, S32, S32) + .clampScalar(1, S32, S64) + .scalarize(0) + .widenScalarToNextPow2(0, 32) + .widenScalarToNextPow2(1, 32); getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF) - .legalFor({{S32, S32}, {S32, S64}}) - .clampScalar(0, S32, S32) - .clampScalar(1, S32, S64) - .scalarize(0) - .widenScalarToNextPow2(0, 32) - .widenScalarToNextPow2(1, 32); + .legalFor({{S32, S32}, {S32, S64}}) + .clampScalar(0, S32, S32) + .clampScalar(1, S32, S64) + .scalarize(0) + .widenScalarToNextPow2(0, 32) + .widenScalarToNextPow2(1, 32); // S64 is only legal on SALU, and needs to be broken into 32-bit elements in // RegBankSelect. From 381dc78286de49f57fbcd6caac7cc2d84387e558 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Tue, 23 Apr 2024 12:09:12 +0100 Subject: [PATCH 06/11] Address review comments. --- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 20e5e42ca1f9a..6e79d717810e9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -4159,18 +4159,18 @@ bool AMDGPULegalizerInfo::legalizeCTLZ_CTTZ(MachineInstr &MI, bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { - auto Dst = MI.getOperand(0).getReg(); - auto Src = MI.getOperand(1).getReg(); - auto DstTy = MRI.getType(Dst); - auto SrcTy = MRI.getType(Src); - auto NumBits = SrcTy.getSizeInBits(); + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(Dst); + LLT SrcTy = MRI.getType(Src); + TypeSize NumBits = SrcTy.getSizeInBits(); assert(NumBits < 32u); auto ShiftAmt = B.buildConstant(S32, 32u - NumBits); - Src = B.buildAnyExt(S32, {Src}).getReg(0u); - Src = B.buildLShr(S32, {Src}, ShiftAmt).getReg(0u); - B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Src}); + auto Tmp = B.buildAnyExt(S32, {Src}).getReg(0u); + Tmp = B.buildLShr(S32, {Tmp}, ShiftAmt).getReg(0u); + B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Tmp}); MI.eraseFromParent(); return true; } From b19d9e290198bec78a9d6303b6e819b51ade4dec Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Thu, 2 May 2024 22:24:12 +0100 Subject: [PATCH 07/11] Address review comments. --- llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 6 +++--- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index d01d0483af071..980e58510ceb7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -3127,9 +3127,9 @@ SDValue AMDGPUTargetLowering::lowerCTLZResults(SDValue Op, assert(isCtlzOpc(Opc)); assert(ResultVT == Arg.getValueType()); - auto const NumBits = ResultVT.getFixedSizeInBits(); - auto NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32); - auto NewOp = SDValue(); + const uint64_t NumBits = ResultVT.getFixedSizeInBits(); + SDValue NumExtBits = DAG.getConstant(32u - NumBits, SL, MVT::i32); + SDValue NewOp; if (Opc == ISD::CTLZ_ZERO_UNDEF) { NewOp = DAG.getNode(ISD::ANY_EXTEND, SL, MVT::i32, Arg); diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 6e79d717810e9..241bdd743d7e5 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -4161,16 +4161,15 @@ bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, MachineIRBuilder &B) const { Register Dst = MI.getOperand(0).getReg(); Register Src = MI.getOperand(1).getReg(); - LLT DstTy = MRI.getType(Dst); LLT SrcTy = MRI.getType(Src); TypeSize NumBits = SrcTy.getSizeInBits(); assert(NumBits < 32u); auto ShiftAmt = B.buildConstant(S32, 32u - NumBits); - auto Tmp = B.buildAnyExt(S32, {Src}).getReg(0u); - Tmp = B.buildLShr(S32, {Tmp}, ShiftAmt).getReg(0u); - B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Tmp}); + auto Extend = B.buildAnyExt(S32, {Src}).getReg(0u); + auto Shift = B.buildLShr(S32, {Extend}, ShiftAmt).getReg(0u); + B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Shift}); MI.eraseFromParent(); return true; } From 375226a2ef32ac5a4938df4cef6914f8e6000137 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Wed, 8 May 2024 11:57:25 +0100 Subject: [PATCH 08/11] Address comments. --- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 241bdd743d7e5..ddd91bef10cda 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -1272,7 +1272,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // The 64-bit versions produce 32-bit results, but only on the SALU. getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF) .legalFor({{S32, S32}, {S32, S64}}) - .customFor({{S32, S8}, {S32, S16}}) + .customIf(scalarNarrowerThan(0, 32)) .clampScalar(0, S32, S32) .clampScalar(1, S32, S64) .scalarize(0) @@ -4168,7 +4168,7 @@ bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, auto ShiftAmt = B.buildConstant(S32, 32u - NumBits); auto Extend = B.buildAnyExt(S32, {Src}).getReg(0u); - auto Shift = B.buildLShr(S32, {Extend}, ShiftAmt).getReg(0u); + auto Shift = B.buildLShr(S32, {Extend}, ShiftAmt); B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Shift}); MI.eraseFromParent(); return true; From f9afbe93e75142c67e8f38b5692dd074efff4e6d Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Thu, 9 May 2024 16:00:20 +0100 Subject: [PATCH 09/11] Address review comments. --- llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 5 +++-- .../AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir | 14 ++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index ddd91bef10cda..15a4b6796880f 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -1272,7 +1272,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // The 64-bit versions produce 32-bit results, but only on the SALU. getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF) .legalFor({{S32, S32}, {S32, S64}}) - .customIf(scalarNarrowerThan(0, 32)) + .customIf(scalarNarrowerThan(1, 32)) .clampScalar(0, S32, S32) .clampScalar(1, S32, S64) .scalarize(0) @@ -4169,7 +4169,8 @@ bool AMDGPULegalizerInfo::legalizeCTLZ_ZERO_UNDEF(MachineInstr &MI, auto ShiftAmt = B.buildConstant(S32, 32u - NumBits); auto Extend = B.buildAnyExt(S32, {Src}).getReg(0u); auto Shift = B.buildLShr(S32, {Extend}, ShiftAmt); - B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {Dst}, {Shift}); + auto Ctlz = B.buildInstr(AMDGPU::G_AMDGPU_FFBH_U32, {S32}, {Shift}); + B.buildTrunc(Dst, Ctlz); MI.eraseFromParent(); return true; } diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir index 32af5e1975a6c..7748b481cf5b7 100644 --- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-ctlz-zero-undef.mir @@ -174,14 +174,12 @@ body: | ; CHECK: liveins: $vgpr0 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 - ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 127 - ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[COPY]], [[C]] - ; CHECK-NEXT: [[CTLZ_ZERO_UNDEF:%[0-9]+]]:_(s32) = G_CTLZ_ZERO_UNDEF [[AND]](s32) - ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 25 - ; CHECK-NEXT: [[SUB:%[0-9]+]]:_(s32) = G_SUB [[CTLZ_ZERO_UNDEF]], [[C1]] - ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY [[SUB]](s32) - ; CHECK-NEXT: [[AND1:%[0-9]+]]:_(s32) = G_AND [[COPY1]], [[C]] - ; CHECK-NEXT: $vgpr0 = COPY [[AND1]](s32) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 25 + ; CHECK-NEXT: [[LSHR:%[0-9]+]]:_(s32) = G_LSHR [[COPY]], [[C]](s32) + ; CHECK-NEXT: [[FFBH:%[0-9]+]]:_(s32) = G_AMDGPU_FFBH_U32 [[LSHR]](s32) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 127 + ; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[FFBH]], [[C1]] + ; CHECK-NEXT: $vgpr0 = COPY [[AND]](s32) %0:_(s32) = COPY $vgpr0 %1:_(s7) = G_TRUNC %0 %2:_(s7) = G_CTLZ_ZERO_UNDEF %1 From 1b08b1446a19ad3690ae95b987bdf123c067e446 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Fri, 10 May 2024 14:47:22 +0100 Subject: [PATCH 10/11] Address review comments. --- llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 217 ++++++++------------ 1 file changed, 90 insertions(+), 127 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll index 9204eb5fdf8ca..b6e53c9e55174 100644 --- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll +++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll @@ -2186,10 +2186,9 @@ define i7 @v_ctlz_zero_undef_i7(i7 %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i7: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 25, v0 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 25, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call i7 @llvm.ctlz.i7(i7 %val, i1 true) ret i7 %ctlz } @@ -2276,19 +2275,18 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i18(ptr addrspace(1) noalias %out, ; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i18: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c -; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 -; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0 -; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-GISEL-NEXT: s_and_b32 s0, s4, 0x3ffff -; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0 -; GFX9-GISEL-NEXT: s_sub_i32 s0, s0, 14 -; GFX9-GISEL-NEXT: s_and_b32 s0, s0, 0x3ffff -; GFX9-GISEL-NEXT: s_lshr_b32 s1, s0, 16 -; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0 -; GFX9-GISEL-NEXT: global_store_short v0, v1, s[2:3] -; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1 -; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[2:3] offset:2 -; GFX9-GISEL-NEXT: s_endpgm +; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-GISEL-NEXT: s_lshr_b32 s0, s4, 14 +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0 +; GFX9-GISEL-NEXT: s_and_b32 s0, s0, 0x3ffff +; GFX9-GISEL-NEXT: s_lshr_b32 s1, s0, 16 +; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-GISEL-NEXT: global_store_short v0, v1, s[2:3] +; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[2:3] offset:2 +; GFX9-GISEL-NEXT: s_endpgm %ctlz = call i18 @llvm.ctlz.i18(i18 %val, i1 true) nounwind readnone store i18 %ctlz, ptr addrspace(1) %out, align 4 ret void @@ -2319,10 +2317,9 @@ define i18 @v_ctlz_zero_undef_i18(i18 %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_i18: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x3ffff, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 14, v0 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 14, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call i18 @llvm.ctlz.i18(i18 %val, i1 true) ret i18 %ctlz } @@ -2358,13 +2355,11 @@ define <2 x i18> @v_ctlz_zero_undef_v2i18(<2 x i18> %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i18: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x3ffff, v0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x3ffff, v1 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 14, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 14, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 14, v0 +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 14, v1 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i18> @llvm.ctlz.v2i18(<2 x i18> %val, i1 true) ret <2 x i18> %ctlz } @@ -2373,17 +2368,13 @@ define <2 x i16> @v_ctlz_zero_undef_v2i16(<2 x i16> %val) { ; SI-LABEL: v_ctlz_zero_undef_v2i16: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; SI-NEXT: v_ffbh_u32_e32 v1, v1 -; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_add_i32_e32 v1, vcc, -16, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0 -; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; SI-NEXT: v_or_b32_e32 v0, v0, v2 -; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; SI-NEXT: s_setpc_b64 s[30:31] +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; SI-NEXT: v_ffbh_u32_e32 v0, v0 +; SI-NEXT: v_or_b32_e32 v0, v0, v2 +; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v2i16: ; VI: ; %bb.0: @@ -2403,13 +2394,11 @@ define <2 x i16> @v_ctlz_zero_undef_v2i16(<2 x i16> %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i16: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 16, v1 ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 16, v0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %val, i1 true) ret <2 x i16> %ctlz } @@ -2418,22 +2407,17 @@ define <3 x i16> @v_ctlz_zero_undef_v3i16(<3 x i16> %val) { ; SI-LABEL: v_ctlz_zero_undef_v3i16: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; SI-NEXT: v_ffbh_u32_e32 v1, v1 -; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_ffbh_u32_e32 v2, v2 ; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0 -; SI-NEXT: v_add_i32_e32 v3, vcc, -16, v2 -; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; SI-NEXT: v_and_b32_e32 v2, 0xffff, v3 -; SI-NEXT: v_or_b32_e32 v0, v1, v0 -; SI-NEXT: v_add_i32_e32 v0, vcc, 0xfff00000, v0 -; SI-NEXT: v_or_b32_e32 v2, 0x100000, v2 -; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16 -; SI-NEXT: s_setpc_b64 s[30:31] +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_ffbh_u32_e32 v0, v0 +; SI-NEXT: v_ffbh_u32_e32 v3, v2 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_or_b32_e32 v2, 0x200000, v3 +; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16 +; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v3i16: ; VI: ; %bb.0: @@ -2455,15 +2439,12 @@ define <3 x i16> @v_ctlz_zero_undef_v3i16(<3 x i16> %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v3i16: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 16, v2 ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 16, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 16, v1 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <3 x i16> @llvm.ctlz.v3i16(<3 x i16> %val, i1 true) ret <3 x i16> %ctlz } @@ -2472,27 +2453,21 @@ define <4 x i16> @v_ctlz_zero_undef_v4i16(<4 x i16> %val) { ; SI-LABEL: v_ctlz_zero_undef_v4i16: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_and_b32_e32 v3, 0xffff, v3 -; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; SI-NEXT: v_ffbh_u32_e32 v3, v3 -; SI-NEXT: v_ffbh_u32_e32 v2, v2 -; SI-NEXT: v_ffbh_u32_e32 v1, v1 -; SI-NEXT: v_ffbh_u32_e32 v0, v0 ; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; SI-NEXT: v_add_i32_e32 v2, vcc, -16, v2 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_add_i32_e32 v0, vcc, -16, v0 -; SI-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; SI-NEXT: v_or_b32_e32 v2, v3, v2 -; SI-NEXT: v_or_b32_e32 v0, v1, v0 -; SI-NEXT: v_add_i32_e32 v2, vcc, 0xfff00000, v2 -; SI-NEXT: v_add_i32_e32 v0, vcc, 0xfff00000, v0 -; SI-NEXT: v_alignbit_b32 v1, v2, v0, 16 -; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2 -; SI-NEXT: s_setpc_b64 s[30:31] +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_ffbh_u32_e32 v3, v3 +; SI-NEXT: v_ffbh_u32_e32 v2, v2 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_ffbh_u32_e32 v0, v0 +; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_or_b32_e32 v2, v2, v3 +; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_alignbit_b32 v1, v2, v0, 16 +; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v4i16: ; VI: ; %bb.0: @@ -2517,19 +2492,14 @@ define <4 x i16> @v_ctlz_zero_undef_v4i16(<4 x i16> %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v4i16: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v2, 16, v2 ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 16, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v3, 16, v3 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 16, v1 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v2 -; GFX9-GISEL-NEXT: v_and_b32_e32 v2, 0xffff, v3 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, v1, 16, v2 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %val, i1 true) ret <4 x i16> %ctlz } @@ -2538,28 +2508,25 @@ define <2 x i8> @v_ctlz_zero_undef_v2i8(<2 x i8> %val) { ; SI-LABEL: v_ctlz_zero_undef_v2i8: ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; SI-NEXT: v_and_b32_e32 v1, 0xff, v1 -; SI-NEXT: v_and_b32_e32 v0, 0xff, v0 +; SI-NEXT: v_lshlrev_b32_e32 v1, 24, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 24, v0 ; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v2, 8, v1 ; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v1, 8, v1 -; SI-NEXT: v_subrev_i32_e32 v0, vcc, 24, v0 -; SI-NEXT: v_and_b32_e32 v0, 0xff, v0 -; SI-NEXT: v_or_b32_e32 v0, v1, v0 -; SI-NEXT: v_add_i32_e32 v0, vcc, 0xffffe800, v0 -; SI-NEXT: v_bfe_u32 v1, v0, 8, 8 +; SI-NEXT: v_or_b32_e32 v0, v0, v2 ; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v2i8: ; VI: ; %bb.0: ; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; VI-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; VI-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; VI-NEXT: v_add_u16_e32 v1, 0xe800, v1 -; VI-NEXT: v_subrev_u16_e32 v0, 24, v0 -; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD -; VI-NEXT: v_lshrrev_b16_e32 v1, 8, v1 -; VI-NEXT: s_setpc_b64 s[30:31] +; VI-NEXT: v_lshlrev_b32_e32 v1, 24, v1 +; VI-NEXT: v_ffbh_u32_e32 v1, v1 +; VI-NEXT: v_lshlrev_b32_e32 v0, 24, v0 +; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v1 +; VI-NEXT: v_ffbh_u32_e32 v0, v0 +; VI-NEXT: v_or_b32_e32 v0, v0, v2 +; VI-NEXT: v_and_b32_e32 v1, 0xff, v1 +; VI-NEXT: s_setpc_b64 s[30:31] ; ; EG-LABEL: v_ctlz_zero_undef_v2i8: ; EG: ; %bb.0: @@ -2569,11 +2536,9 @@ define <2 x i8> @v_ctlz_zero_undef_v2i8(<2 x i8> %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i8: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 24, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 24, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %val, i1 true) ret <2 x i8> %ctlz } @@ -2614,13 +2579,11 @@ define <2 x i7> @v_ctlz_zero_undef_v2i7(<2 x i7> %val) { ; GFX9-GISEL-LABEL: v_ctlz_zero_undef_v2i7: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0x7f, v0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0x7f, v1 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v0, 25, v0 -; GFX9-GISEL-NEXT: v_subrev_u32_e32 v1, 25, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 25, v0 +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 25, v1 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i7> @llvm.ctlz.v2i7(<2 x i7> %val, i1 true) ret <2 x i7> %ctlz } From f98d4c791dad57913e266030c20de3bae4e43b83 Mon Sep 17 00:00:00 2001 From: Leon Clark Date: Fri, 17 May 2024 22:36:52 +0100 Subject: [PATCH 11/11] Fix indentation. --- llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll | 158 ++++++++++---------- 1 file changed, 79 insertions(+), 79 deletions(-) diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll index b6e53c9e55174..d94a27e8c0200 100644 --- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll +++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_undef.ll @@ -2187,8 +2187,8 @@ define i7 @v_ctlz_zero_undef_i7(i7 %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 25, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call i7 @llvm.ctlz.i7(i7 %val, i1 true) ret i7 %ctlz } @@ -2275,18 +2275,18 @@ define amdgpu_kernel void @s_ctlz_zero_undef_i18(ptr addrspace(1) noalias %out, ; GFX9-GISEL-LABEL: s_ctlz_zero_undef_i18: ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_load_dword s4, s[0:1], 0x2c -; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 -; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0 -; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-GISEL-NEXT: s_lshr_b32 s0, s4, 14 -; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0 -; GFX9-GISEL-NEXT: s_and_b32 s0, s0, 0x3ffff -; GFX9-GISEL-NEXT: s_lshr_b32 s1, s0, 16 -; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0 -; GFX9-GISEL-NEXT: global_store_short v0, v1, s[2:3] -; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1 -; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[2:3] offset:2 -; GFX9-GISEL-NEXT: s_endpgm +; GFX9-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24 +; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0 +; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-GISEL-NEXT: s_lshr_b32 s0, s4, 14 +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s0, s0 +; GFX9-GISEL-NEXT: s_and_b32 s0, s0, 0x3ffff +; GFX9-GISEL-NEXT: s_lshr_b32 s1, s0, 16 +; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s0 +; GFX9-GISEL-NEXT: global_store_short v0, v1, s[2:3] +; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s1 +; GFX9-GISEL-NEXT: global_store_byte v0, v1, s[2:3] offset:2 +; GFX9-GISEL-NEXT: s_endpgm %ctlz = call i18 @llvm.ctlz.i18(i18 %val, i1 true) nounwind readnone store i18 %ctlz, ptr addrspace(1) %out, align 4 ret void @@ -2318,8 +2318,8 @@ define i18 @v_ctlz_zero_undef_i18(i18 %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 14, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call i18 @llvm.ctlz.i18(i18 %val, i1 true) ret i18 %ctlz } @@ -2356,10 +2356,10 @@ define <2 x i18> @v_ctlz_zero_undef_v2i18(<2 x i18> %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 14, v0 -; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 14, v1 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 14, v1 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i18> @llvm.ctlz.v2i18(<2 x i18> %val, i1 true) ret <2 x i18> %ctlz } @@ -2369,12 +2369,12 @@ define <2 x i16> @v_ctlz_zero_undef_v2i16(<2 x i16> %val) { ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; SI-NEXT: v_ffbh_u32_e32 v1, v1 -; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1 -; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_or_b32_e32 v0, v0, v2 -; SI-NEXT: s_setpc_b64 s[30:31] +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v1 +; SI-NEXT: v_ffbh_u32_e32 v0, v0 +; SI-NEXT: v_or_b32_e32 v0, v0, v2 +; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v2i16: ; VI: ; %bb.0: @@ -2395,10 +2395,10 @@ define <2 x i16> @v_ctlz_zero_undef_v2i16(<2 x i16> %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i16> @llvm.ctlz.v2i16(<2 x i16> %val, i1 true) ret <2 x i16> %ctlz } @@ -2408,16 +2408,16 @@ define <3 x i16> @v_ctlz_zero_undef_v3i16(<3 x i16> %val) { ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; SI-NEXT: v_ffbh_u32_e32 v1, v1 -; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_ffbh_u32_e32 v3, v2 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 -; SI-NEXT: v_or_b32_e32 v2, 0x200000, v3 -; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16 -; SI-NEXT: s_setpc_b64 s[30:31] +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_ffbh_u32_e32 v0, v0 +; SI-NEXT: v_ffbh_u32_e32 v3, v2 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_or_b32_e32 v2, 0x200000, v3 +; SI-NEXT: v_alignbit_b32 v1, v3, v0, 16 +; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v3i16: ; VI: ; %bb.0: @@ -2440,11 +2440,11 @@ define <3 x i16> @v_ctlz_zero_undef_v3i16(<3 x i16> %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <3 x i16> @llvm.ctlz.v3i16(<3 x i16> %val, i1 true) ret <3 x i16> %ctlz } @@ -2454,20 +2454,20 @@ define <4 x i16> @v_ctlz_zero_undef_v4i16(<4 x i16> %val) { ; SI: ; %bb.0: ; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 -; SI-NEXT: v_ffbh_u32_e32 v3, v3 -; SI-NEXT: v_ffbh_u32_e32 v2, v2 -; SI-NEXT: v_ffbh_u32_e32 v1, v1 -; SI-NEXT: v_ffbh_u32_e32 v0, v0 -; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 -; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 -; SI-NEXT: v_or_b32_e32 v2, v2, v3 -; SI-NEXT: v_or_b32_e32 v0, v0, v1 -; SI-NEXT: v_alignbit_b32 v1, v2, v0, 16 -; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2 -; SI-NEXT: s_setpc_b64 s[30:31] +; SI-NEXT: v_lshlrev_b32_e32 v2, 16, v2 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0 +; SI-NEXT: v_ffbh_u32_e32 v3, v3 +; SI-NEXT: v_ffbh_u32_e32 v2, v2 +; SI-NEXT: v_ffbh_u32_e32 v1, v1 +; SI-NEXT: v_ffbh_u32_e32 v0, v0 +; SI-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1 +; SI-NEXT: v_or_b32_e32 v2, v2, v3 +; SI-NEXT: v_or_b32_e32 v0, v0, v1 +; SI-NEXT: v_alignbit_b32 v1, v2, v0, 16 +; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v2 +; SI-NEXT: s_setpc_b64 s[30:31] ; ; VI-LABEL: v_ctlz_zero_undef_v4i16: ; VI: ; %bb.0: @@ -2493,13 +2493,13 @@ define <4 x i16> @v_ctlz_zero_undef_v4i16(<4 x i16> %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 -; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 -; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 -; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 +; GFX9-GISEL-NEXT: s_flbit_i32_b32 s4, 0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX9-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v0, s4, 16, v0 +; GFX9-GISEL-NEXT: v_lshl_or_b32 v1, s4, 16, v1 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %val, i1 true) ret <4 x i16> %ctlz } @@ -2520,13 +2520,13 @@ define <2 x i8> @v_ctlz_zero_undef_v2i8(<2 x i8> %val) { ; VI: ; %bb.0: ; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; VI-NEXT: v_lshlrev_b32_e32 v1, 24, v1 -; VI-NEXT: v_ffbh_u32_e32 v1, v1 -; VI-NEXT: v_lshlrev_b32_e32 v0, 24, v0 -; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v1 -; VI-NEXT: v_ffbh_u32_e32 v0, v0 -; VI-NEXT: v_or_b32_e32 v0, v0, v2 -; VI-NEXT: v_and_b32_e32 v1, 0xff, v1 -; VI-NEXT: s_setpc_b64 s[30:31] +; VI-NEXT: v_ffbh_u32_e32 v1, v1 +; VI-NEXT: v_lshlrev_b32_e32 v0, 24, v0 +; VI-NEXT: v_lshlrev_b16_e32 v2, 8, v1 +; VI-NEXT: v_ffbh_u32_e32 v0, v0 +; VI-NEXT: v_or_b32_e32 v0, v0, v2 +; VI-NEXT: v_and_b32_e32 v1, 0xff, v1 +; VI-NEXT: s_setpc_b64 s[30:31] ; ; EG-LABEL: v_ctlz_zero_undef_v2i8: ; EG: ; %bb.0: @@ -2537,8 +2537,8 @@ define <2 x i8> @v_ctlz_zero_undef_v2i8(<2 x i8> %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 -; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_ffbh_u32_sdwa v1, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i8> @llvm.ctlz.v2i8(<2 x i8> %val, i1 true) ret <2 x i8> %ctlz } @@ -2580,10 +2580,10 @@ define <2 x i7> @v_ctlz_zero_undef_v2i7(<2 x i7> %val) { ; GFX9-GISEL: ; %bb.0: ; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) ; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v0, 25, v0 -; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 25, v1 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 -; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 -; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 25, v1 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v0, v0 +; GFX9-GISEL-NEXT: v_ffbh_u32_e32 v1, v1 +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] %ctlz = call <2 x i7> @llvm.ctlz.v2i7(<2 x i7> %val, i1 true) ret <2 x i7> %ctlz }