Skip to content

Commit

Permalink
SelectionDAG: Implement expansion of {S,U}MIN/MAX in integer legaliza…
Browse files Browse the repository at this point in the history 
…tion

Fixes {u,}long_{min,max,clamp} opencl piglit regressions on EG.

Reviewers: arsenm
Differential Revision: http://reviews.llvm.org/D17898

llvm-svn: 272272
  • Loading branch information
@jvesely
jvesely committed Jun 9, 2016
1 parent 5b89fbc commit 2da0cba
Show file tree
Hide file tree
Showing 5 changed files with 341 additions and 101 deletions.
53 changes: 53 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
View file
Expand Up @@ -1371,6 +1371,11 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::OR:
case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;

case ISD::UMAX:
case ISD::SMAX:
case ISD::UMIN:
case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;

case ISD::ADD:
case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;

Expand Down Expand Up @@ -1663,6 +1668,54 @@ ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
}
}

static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {

switch (Op) {
default: llvm_unreachable("invalid min/max opcode");
case ISD::SMAX:
return std::make_pair(ISD::SETGT, ISD::UMAX);
case ISD::UMAX:
return std::make_pair(ISD::SETUGT, ISD::UMAX);
case ISD::SMIN:
return std::make_pair(ISD::SETLT, ISD::UMIN);
case ISD::UMIN:
return std::make_pair(ISD::SETULT, ISD::UMIN);
}
}

void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
SDValue &Lo, SDValue &Hi) {
SDLoc DL(N);
ISD::NodeType LoOpc;
ISD::CondCode CondC;
std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());

// Expand the subcomponents.
SDValue LHSL, LHSH, RHSL, RHSH;
GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
GetExpandedInteger(N->getOperand(1), RHSL, RHSH);

// Value types
EVT NVT = LHSL.getValueType();
EVT CCT = getSetCCResultType(NVT);

// Hi part is always the same op
Hi = DAG.getNode(N->getOpcode(), DL, {NVT, NVT}, {LHSH, RHSH});

// We need to know whether to select Lo part that corresponds to 'winning'
// Hi part or if Hi parts are equal.
SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);

// Lo part corresponding to the 'winning' Hi part
SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);

// Recursed Lo part if Hi parts are equal, this uses unsigned version
SDValue LoMinMax = DAG.getNode(LoOpc, DL, {NVT, NVT}, {LHSL, RHSL});

Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
}

void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
SDValue &Lo, SDValue &Hi) {
SDLoc dl(N);
Expand Down
2 changes: 2 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
View file
Expand Up @@ -361,6 +361,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);

void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);

void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
Expand Down
169 changes: 119 additions & 50 deletions llvm/test/CodeGen/AMDGPU/max.ll
View file
@@ -1,19 +1,17 @@
; RUN: llc -march=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
; RUN: llc -march=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s

declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone

; FUNC-LABEL: {{^}}v_test_imax_sge_i32:
; SI: v_max_i32_e32

; EG: MAX_INT
define void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
%a = load i32, i32 addrspace(1)* %gep0, align 4
%b = load i32, i32 addrspace(1)* %gep1, align 4
%a = load i32, i32 addrspace(1)* %aptr, align 4
%b = load i32, i32 addrspace(1)* %bptr, align 4
%cmp = icmp sge i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
store i32 %val, i32 addrspace(1)* %outgep, align 4
store i32 %val, i32 addrspace(1)* %out, align 4
ret void
}

Expand All @@ -22,21 +20,25 @@ define void @v_test_imax_sge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr
; SI: v_max_i32_e32
; SI: v_max_i32_e32
; SI: v_max_i32_e32

; These could be merged into one
; EG: MAX_INT
; EG: MAX_INT
; EG: MAX_INT
; EG: MAX_INT
define void @v_test_imax_sge_v4i32(<4 x i32> addrspace(1)* %out, <4 x i32> addrspace(1)* %aptr, <4 x i32> addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr <4 x i32>, <4 x i32> addrspace(1)* %out, i32 %tid
%a = load <4 x i32>, <4 x i32> addrspace(1)* %gep0, align 4
%b = load <4 x i32>, <4 x i32> addrspace(1)* %gep1, align 4
%a = load <4 x i32>, <4 x i32> addrspace(1)* %aptr, align 4
%b = load <4 x i32>, <4 x i32> addrspace(1)* %bptr, align 4
%cmp = icmp sge <4 x i32> %a, %b
%val = select <4 x i1> %cmp, <4 x i32> %a, <4 x i32> %b
store <4 x i32> %val, <4 x i32> addrspace(1)* %outgep, align 4
store <4 x i32> %val, <4 x i32> addrspace(1)* %out, align 4
ret void
}

; FUNC-LABEL: @s_test_imax_sge_i32
; SI: s_max_i32

; EG: MAX_INT
define void @s_test_imax_sge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
%cmp = icmp sge i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
Expand All @@ -46,6 +48,8 @@ define void @s_test_imax_sge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwin

; FUNC-LABEL: {{^}}s_test_imax_sge_imm_i32:
; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9

; EG: MAX_INT {{.*}}literal.{{[xyzw]}}
define void @s_test_imax_sge_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
%cmp = icmp sge i32 %a, 9
%val = select i1 %cmp, i32 %a, i32 9
Expand All @@ -57,21 +61,21 @@ define void @s_test_imax_sge_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
; SI: buffer_load_sbyte
; SI: buffer_load_sbyte
; SI: v_max_i32_e32

; EG: MAX_INT
define void @v_test_imax_sge_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %aptr, i8 addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr i8, i8 addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr i8, i8 addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr i8, i8 addrspace(1)* %out, i32 %tid
%a = load i8, i8 addrspace(1)* %gep0, align 1
%b = load i8, i8 addrspace(1)* %gep1, align 1
%a = load i8, i8 addrspace(1)* %aptr, align 1
%b = load i8, i8 addrspace(1)* %bptr, align 1
%cmp = icmp sge i8 %a, %b
%val = select i1 %cmp, i8 %a, i8 %b
store i8 %val, i8 addrspace(1)* %outgep, align 1
store i8 %val, i8 addrspace(1)* %out, align 1
ret void
}

; FUNC-LABEL: {{^}}s_test_imax_sgt_imm_i32:
; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9

; EG: MAX_INT {{.*}}literal.{{[xyzw]}}
define void @s_test_imax_sgt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
%cmp = icmp sgt i32 %a, 9
%val = select i1 %cmp, i32 %a, i32 9
Expand All @@ -82,29 +86,33 @@ define void @s_test_imax_sgt_imm_i32(i32 addrspace(1)* %out, i32 %a) nounwind {
; FUNC-LABEL: {{^}}s_test_imax_sgt_imm_v2i32:
; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9
; SI: s_max_i32 {{s[0-9]+}}, {{s[0-9]+}}, 9

; EG: MAX_INT {{.*}}literal.{{[xyzw]}}
; EG: MAX_INT {{.*}}literal.{{[xyzw]}}
define void @s_test_imax_sgt_imm_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a) nounwind {
%cmp = icmp sgt <2 x i32> %a, <i32 9, i32 9>
%val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> <i32 9, i32 9>
store <2 x i32> %val, <2 x i32> addrspace(1)* %out, align 4
ret void
}

; FUNC-LABEL: @v_test_imax_sgt_i32
; SI: v_max_i32_e32

; EG: MAX_INT
define void @v_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
%a = load i32, i32 addrspace(1)* %gep0, align 4
%b = load i32, i32 addrspace(1)* %gep1, align 4
%a = load i32, i32 addrspace(1)* %aptr, align 4
%b = load i32, i32 addrspace(1)* %bptr, align 4
%cmp = icmp sgt i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
store i32 %val, i32 addrspace(1)* %outgep, align 4
store i32 %val, i32 addrspace(1)* %out, align 4
ret void
}

; FUNC-LABEL: @s_test_imax_sgt_i32
; SI: s_max_i32

; EG: MAX_INT
define void @s_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
%cmp = icmp sgt i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
Expand All @@ -114,21 +122,21 @@ define void @s_test_imax_sgt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwin

; FUNC-LABEL: @v_test_umax_uge_i32
; SI: v_max_u32_e32

; EG: MAX_UINT
define void @v_test_umax_uge_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
%a = load i32, i32 addrspace(1)* %gep0, align 4
%b = load i32, i32 addrspace(1)* %gep1, align 4
%a = load i32, i32 addrspace(1)* %aptr, align 4
%b = load i32, i32 addrspace(1)* %bptr, align 4
%cmp = icmp uge i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
store i32 %val, i32 addrspace(1)* %outgep, align 4
store i32 %val, i32 addrspace(1)* %out, align 4
ret void
}

; FUNC-LABEL: @s_test_umax_uge_i32
; SI: s_max_u32

; EG: MAX_UINT
define void @s_test_umax_uge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
%cmp = icmp uge i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
Expand All @@ -142,6 +150,11 @@ define void @s_test_umax_uge_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwin
; SI: s_max_u32
; SI-NOT: s_max_u32
; SI: s_endpgm

; EG: MAX_UINT
; EG: MAX_UINT
; EG: MAX_UINT
; EG-NOT: MAX_UINT
define void @s_test_umax_uge_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a, <3 x i32> %b) nounwind {
%cmp = icmp uge <3 x i32> %a, %b
%val = select <3 x i1> %cmp, <3 x i32> %a, <3 x i32> %b
Expand All @@ -153,36 +166,34 @@ define void @s_test_umax_uge_v3i32(<3 x i32> addrspace(1)* %out, <3 x i32> %a, <
; SI: buffer_load_ubyte
; SI: buffer_load_ubyte
; SI: v_max_u32_e32

; EG: MAX_UINT
define void @v_test_umax_uge_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %aptr, i8 addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr i8, i8 addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr i8, i8 addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr i8, i8 addrspace(1)* %out, i32 %tid
%a = load i8, i8 addrspace(1)* %gep0, align 1
%b = load i8, i8 addrspace(1)* %gep1, align 1
%a = load i8, i8 addrspace(1)* %aptr, align 1
%b = load i8, i8 addrspace(1)* %bptr, align 1
%cmp = icmp uge i8 %a, %b
%val = select i1 %cmp, i8 %a, i8 %b
store i8 %val, i8 addrspace(1)* %outgep, align 1
store i8 %val, i8 addrspace(1)* %out, align 1
ret void
}

; FUNC-LABEL: @v_test_umax_ugt_i32
; SI: v_max_u32_e32

; EG: MAX_UINT
define void @v_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %aptr, i32 addrspace(1)* %bptr) nounwind {
%tid = call i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
%gep0 = getelementptr i32, i32 addrspace(1)* %aptr, i32 %tid
%gep1 = getelementptr i32, i32 addrspace(1)* %bptr, i32 %tid
%outgep = getelementptr i32, i32 addrspace(1)* %out, i32 %tid
%a = load i32, i32 addrspace(1)* %gep0, align 4
%b = load i32, i32 addrspace(1)* %gep1, align 4
%a = load i32, i32 addrspace(1)* %aptr, align 4
%b = load i32, i32 addrspace(1)* %bptr, align 4
%cmp = icmp ugt i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
store i32 %val, i32 addrspace(1)* %outgep, align 4
store i32 %val, i32 addrspace(1)* %out, align 4
ret void
}

; FUNC-LABEL: {{^}}s_test_umax_ugt_i32:
; SI: s_max_u32

; EG: MAX_UINT
define void @s_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwind {
%cmp = icmp ugt i32 %a, %b
%val = select i1 %cmp, i32 %a, i32 %b
Expand All @@ -193,6 +204,9 @@ define void @s_test_umax_ugt_i32(i32 addrspace(1)* %out, i32 %a, i32 %b) nounwin
; FUNC-LABEL: {{^}}s_test_umax_ugt_imm_v2i32:
; SI-DAG: s_max_u32 {{s[0-9]+}}, {{s[0-9]+}}, 15
; SI-DAG: s_max_u32 {{s[0-9]+}}, {{s[0-9]+}}, 23

; EG: MAX_UINT {{.*}}literal.{{[xyzw]}}
; EG: MAX_UINT {{.*}}literal.{{[xyzw]}}
define void @s_test_umax_ugt_imm_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %a) nounwind {
%cmp = icmp ugt <2 x i32> %a, <i32 15, i32 23>
%val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> <i32 15, i32 23>
Expand All @@ -207,6 +221,8 @@ define void @s_test_umax_ugt_imm_v2i32(<2 x i32> addrspace(1)* %out, <2 x i32> %
; SI: s_max_u32 [[MAX:s[0-9]+]], [[A]], [[B]]
; SI: v_mov_b32_e32 [[VMAX:v[0-9]+]], [[MAX]]
; SI: buffer_store_dword [[VMAX]]

; EG: MAX_UINT
define void @simplify_demanded_bits_test_umax_ugt_i16(i32 addrspace(1)* %out, i16 zeroext %a, i16 zeroext %b) nounwind {
%a.ext = zext i16 %a to i32
%b.ext = zext i16 %b to i32
Expand All @@ -225,6 +241,8 @@ define void @simplify_demanded_bits_test_umax_ugt_i16(i32 addrspace(1)* %out, i1
; SI: s_max_i32 [[MAX:s[0-9]+]], [[A]], [[B]]
; SI: v_mov_b32_e32 [[VMAX:v[0-9]+]], [[MAX]]
; SI: buffer_store_dword [[VMAX]]

; EG: MAX_INT
define void @simplify_demanded_bits_test_max_slt_i16(i32 addrspace(1)* %out, i16 signext %a, i16 signext %b) nounwind {
%a.ext = sext i16 %a to i32
%b.ext = sext i16 %b to i32
Expand All @@ -242,9 +260,60 @@ define void @simplify_demanded_bits_test_max_slt_i16(i32 addrspace(1)* %out, i16
; SI: s_sext_i32_i16
; SI: s_sext_i32_i16
; SI: s_max_i32

; EG: MAX_INT
define void @s_test_imax_sge_i16(i16 addrspace(1)* %out, i16 %a, i16 %b) nounwind {
%cmp = icmp sge i16 %a, %b
%val = select i1 %cmp, i16 %a, i16 %b
store i16 %val, i16 addrspace(1)* %out
ret void
}

; 64 bit
; FUNC-LABEL: {{^}}test_umax_ugt_i64
; SI: s_endpgm

; EG: MAX_UINT
; EG: MAX_UINT
define void @test_umax_ugt_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
%tmp = icmp ugt i64 %a, %b
%val = select i1 %tmp, i64 %a, i64 %b
store i64 %val, i64 addrspace(1)* %out, align 8
ret void
}

; FUNC-LABEL: {{^}}test_umax_uge_i64
; SI: s_endpgm

; EG: MAX_UINT
; EG: MAX_UINT
define void @test_umax_uge_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
%tmp = icmp uge i64 %a, %b
%val = select i1 %tmp, i64 %a, i64 %b
store i64 %val, i64 addrspace(1)* %out, align 8
ret void
}

; FUNC-LABEL: {{^}}test_imax_sgt_i64
; SI: s_endpgm

; EG-DAG: MAX_UINT
; EG-DAG: MAX_INT
define void @test_imax_sgt_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
%tmp = icmp sgt i64 %a, %b
%val = select i1 %tmp, i64 %a, i64 %b
store i64 %val, i64 addrspace(1)* %out, align 8
ret void
}

; FUNC-LABEL: {{^}}test_imax_sge_i64
; SI: s_endpgm

; EG-DAG: MAX_UINT
; EG-DAG: MAX_INT
define void @test_imax_sge_i64(i64 addrspace(1)* %out, i64 %a, i64 %b) nounwind {
%tmp = icmp sge i64 %a, %b
%val = select i1 %tmp, i64 %a, i64 %b
store i64 %val, i64 addrspace(1)* %out, align 8
ret void
}

0 comments on commit 2da0cba

Please sign in to comment.