Skip to content

Commit

Permalink
[X86] Custom widen 128/256-bit vXi32 uint_to_fp on avx512f targets wi…
Browse files Browse the repository at this point in the history 
…thout avx512vl. Similar for vXi64 sint_to_fp/uint_to_fp on avx512dq without avx512vl.

Previously we widened these through isel patterns, but that
didn't work for STRICT_ nodes. Those need to be padded with
zeroes in the upper bits which is harder to do in isel patterns.
  • Loading branch information
@topperc
topperc committed Dec 26, 2019
1 parent ad1f7a8 commit f953882
Show file tree
Hide file tree
Showing 5 changed files with 124 additions and 90 deletions.
123 changes: 106 additions & 17 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
View file
Expand Up @@ -1637,10 +1637,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Legal);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Legal);
setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal);
setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Legal);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Legal);
setOperationAction(ISD::UINT_TO_FP, MVT::v8i32,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::UINT_TO_FP, MVT::v4i32,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32,
Subtarget.hasVLX() ? Legal : Custom);

for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
setOperationAction(ISD::SMAX, VT, Legal);
Expand All @@ -1665,10 +1669,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,

if (Subtarget.hasDQI()) {
for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
setOperationAction(ISD::SINT_TO_FP, VT, Legal);
setOperationAction(ISD::UINT_TO_FP, VT, Legal);
setOperationAction(ISD::STRICT_SINT_TO_FP, VT, Legal);
setOperationAction(ISD::STRICT_UINT_TO_FP, VT, Legal);
setOperationAction(ISD::SINT_TO_FP, VT,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::UINT_TO_FP, VT,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::STRICT_SINT_TO_FP, VT,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::STRICT_UINT_TO_FP, VT,
Subtarget.hasVLX() ? Legal : Custom);
setOperationAction(ISD::FP_TO_SINT, VT, Legal);
setOperationAction(ISD::FP_TO_UINT, VT, Legal);
setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Legal);
Expand Down Expand Up @@ -18570,6 +18578,46 @@ static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
DAG.getIntPtrConstant(0, DL));
}

static SDValue lowerINT_TO_FP_vXi64(SDValue Op, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
assert(Subtarget.hasDQI() && !Subtarget.hasVLX() && "Unexpected features");

SDLoc DL(Op);
bool IsStrict = Op->isStrictFPOpcode();
MVT VT = Op->getSimpleValueType(0);
SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
MVT SrcVT = Src.getSimpleValueType();
assert((SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64) &&
"Unsupported custom type");

// With AVX512DQ, but not VLX we need to widen to get a 512-bit result type.
assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&
"Unexpected VT!");
MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;

// Need to concat with zero vector for strict fp to avoid spurious
// exceptions.
SDValue Tmp =
IsStrict ? DAG.getConstant(0, DL, MVT::v8i64) : DAG.getUNDEF(MVT::v8i64);
Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
DAG.getIntPtrConstant(0, DL));
SDValue Res, Chain;
if (IsStrict) {
Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
{Op->getOperand(0), Src});
Chain = Res.getValue(1);
} else {
Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
}

Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
DAG.getIntPtrConstant(0, DL));

if (IsStrict)
return DAG.getMergeValues({Res, Chain}, DL);
return Res;
}

SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
SelectionDAG &DAG) const {
bool IsStrict = Op->isStrictFPOpcode();
Expand All @@ -18596,6 +18644,9 @@ SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
DAG.getUNDEF(SrcVT)));
}
if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64)
return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);

return SDValue();
}

Expand Down Expand Up @@ -18936,6 +18987,49 @@ static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,

static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
SDLoc DL(Op);
bool IsStrict = Op->isStrictFPOpcode();
SDValue V = Op->getOperand(IsStrict ? 1 : 0);
MVT VecIntVT = V.getSimpleValueType();
assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
"Unsupported custom type");

if (Subtarget.hasAVX512()) {
// With AVX512, but not VLX we need to widen to get a 512-bit result type.
assert(!Subtarget.hasVLX() && "Unexpected features");
MVT VT = Op->getSimpleValueType(0);

// v8i32->v8f64 is legal with AVX512 so just return it.
if (VT == MVT::v8f64)
return Op;

assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
"Unexpected VT!");
MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
// Need to concat with zero vector for strict fp to avoid spurious
// exceptions.
SDValue Tmp =
IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
DAG.getIntPtrConstant(0, DL));
SDValue Res, Chain;
if (IsStrict) {
Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
{Op->getOperand(0), V});
Chain = Res.getValue(1);
} else {
Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
}

Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
DAG.getIntPtrConstant(0, DL));

if (IsStrict)
return DAG.getMergeValues({Res, Chain}, DL);
return Res;
}

// The algorithm is the following:
// #ifdef __SSE4_1__
// uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
Expand All @@ -18957,20 +19051,13 @@ static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
if (DAG.getTarget().Options.UnsafeFPMath)
return SDValue();

SDLoc DL(Op);
bool IsStrict = Op->isStrictFPOpcode();
SDValue V = Op->getOperand(IsStrict ? 1 : 0);
MVT VecIntVT = V.getSimpleValueType();
bool Is128 = VecIntVT == MVT::v4i32;
MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
// If we convert to something else than the supported type, e.g., to v4f64,
// abort early.
if (VecFloatVT != Op->getSimpleValueType(0))
return SDValue();

assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
"Unsupported custom type");

// In the #idef/#else code, we have in common:
// - The vector of constants:
// -- 0x4b000000
Expand Down Expand Up @@ -19051,8 +19138,10 @@ static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
return lowerUINT_TO_FP_v2i32(Op, DAG, Subtarget, dl);
case MVT::v4i32:
case MVT::v8i32:
assert(!Subtarget.hasAVX512());
return lowerUINT_TO_FP_vXi32(Op, DAG, Subtarget);
case MVT::v2i64:
case MVT::v4i64:
return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
}
}

Expand All @@ -19064,7 +19153,7 @@ SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
SDLoc dl(Op);
auto PtrVT = getPointerTy(DAG.getDataLayout());
MVT SrcVT = Src.getSimpleValueType();
MVT DstVT = Op.getSimpleValueType();
MVT DstVT = Op->getSimpleValueType(0);
SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();

if (DstVT == MVT::f128)
Expand Down
45 changes: 0 additions & 45 deletions llvm/lib/Target/X86/X86InstrAVX512.td
View file
Expand Up @@ -8382,21 +8382,6 @@ def : Pat<(v4i32 (X86any_cvttp2ui (v4f64 VR256X:$src1))),
(EXTRACT_SUBREG (v8i32 (VCVTTPD2UDQZrr
(v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_xmm)>;

def : Pat<(v8f32 (any_uint_to_fp (v8i32 VR256X:$src1))),
(EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
(v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_ymm)>;

def : Pat<(v4f32 (any_uint_to_fp (v4i32 VR128X:$src1))),
(EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
(v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
VR128X:$src1, sub_xmm)))), sub_xmm)>;

def : Pat<(v4f64 (any_uint_to_fp (v4i32 VR128X:$src1))),
(EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
(v8i32 (INSERT_SUBREG (IMPLICIT_DEF),
VR128X:$src1, sub_xmm)))), sub_ymm)>;
}

let Predicates = [HasVLX] in {
Expand Down Expand Up @@ -8513,36 +8498,6 @@ def : Pat<(v4i64 (X86any_cvttp2ui (v4f64 VR256X:$src1))),
(EXTRACT_SUBREG (v8i64 (VCVTTPD2UQQZrr
(v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_ymm)>;

def : Pat<(v4f32 (any_sint_to_fp (v4i64 VR256X:$src1))),
(EXTRACT_SUBREG (v8f32 (VCVTQQ2PSZrr
(v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_xmm)>;

def : Pat<(v2f64 (any_sint_to_fp (v2i64 VR128X:$src1))),
(EXTRACT_SUBREG (v8f64 (VCVTQQ2PDZrr
(v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
VR128X:$src1, sub_xmm)))), sub_xmm)>;

def : Pat<(v4f64 (any_sint_to_fp (v4i64 VR256X:$src1))),
(EXTRACT_SUBREG (v8f64 (VCVTQQ2PDZrr
(v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_ymm)>;

def : Pat<(v4f32 (any_uint_to_fp (v4i64 VR256X:$src1))),
(EXTRACT_SUBREG (v8f32 (VCVTUQQ2PSZrr
(v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_xmm)>;

def : Pat<(v2f64 (any_uint_to_fp (v2i64 VR128X:$src1))),
(EXTRACT_SUBREG (v8f64 (VCVTUQQ2PDZrr
(v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
VR128X:$src1, sub_xmm)))), sub_xmm)>;

def : Pat<(v4f64 (any_uint_to_fp (v4i64 VR256X:$src1))),
(EXTRACT_SUBREG (v8f64 (VCVTUQQ2PDZrr
(v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
VR256X:$src1, sub_ymm)))), sub_ymm)>;
}

//===----------------------------------------------------------------------===//
Expand Down
12 changes: 4 additions & 8 deletions llvm/test/CodeGen/X86/vec-strict-inttofp-128.ll
View file
Expand Up @@ -537,10 +537,9 @@ define <4 x float> @uitofp_v4i32_v4f32(<4 x i32> %x) #0 {
; AVX1-64-NEXT: vaddps %xmm0, %xmm1, %xmm0
; AVX1-64-NEXT: retq
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512F-LABEL: uitofp_v4i32_v4f32:
; AVX512F: # %bb.0:
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512F-NEXT: vmovaps %xmm0, %xmm0
; AVX512F-NEXT: vcvtudq2ps %zmm0, %zmm0
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
; AVX512F-NEXT: vzeroupper
Expand All @@ -551,10 +550,9 @@ define <4 x float> @uitofp_v4i32_v4f32(<4 x i32> %x) #0 {
; AVX512VL-NEXT: vcvtudq2ps %xmm0, %xmm0
; AVX512VL-NEXT: ret{{[l|q]}}
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: uitofp_v4i32_v4f32:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512DQ-NEXT: vmovaps %xmm0, %xmm0
; AVX512DQ-NEXT: vcvtudq2ps %zmm0, %zmm0
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
; AVX512DQ-NEXT: vzeroupper
Expand Down Expand Up @@ -908,10 +906,9 @@ define <2 x double> @sitofp_v2i64_v2f64(<2 x i64> %x) #0 {
; AVX-64-NEXT: vunpcklpd {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; AVX-64-NEXT: retq
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: sitofp_v2i64_v2f64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512DQ-NEXT: vmovaps %xmm0, %xmm0
; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
; AVX512DQ-NEXT: vzeroupper
Expand Down Expand Up @@ -1014,10 +1011,9 @@ define <2 x double> @uitofp_v2i64_v2f64(<2 x i64> %x) #0 {
; AVX512VL-64-NEXT: vaddpd %xmm0, %xmm1, %xmm0
; AVX512VL-64-NEXT: retq
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: uitofp_v2i64_v2f64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 def $zmm0
; AVX512DQ-NEXT: vmovaps %xmm0, %xmm0
; AVX512DQ-NEXT: vcvtuqq2pd %zmm0, %zmm0
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
; AVX512DQ-NEXT: vzeroupper
Expand Down
18 changes: 6 additions & 12 deletions llvm/test/CodeGen/X86/vec-strict-inttofp-256.ll
View file
Expand Up @@ -163,10 +163,9 @@ define <8 x float> @uitofp_v8i32_v8f32(<8 x i32> %x) #0 {
; AVX1-NEXT: vaddps %ymm0, %ymm1, %ymm0
; AVX1-NEXT: ret{{[l|q]}}
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512F-LABEL: uitofp_v8i32_v8f32:
; AVX512F: # %bb.0:
; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512F-NEXT: vmovaps %ymm0, %ymm0
; AVX512F-NEXT: vcvtudq2ps %zmm0, %zmm0
; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512F-NEXT: ret{{[l|q]}}
Expand All @@ -176,10 +175,9 @@ define <8 x float> @uitofp_v8i32_v8f32(<8 x i32> %x) #0 {
; AVX512VL-NEXT: vcvtudq2ps %ymm0, %ymm0
; AVX512VL-NEXT: ret{{[l|q]}}
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: uitofp_v8i32_v8f32:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512DQ-NEXT: vmovaps %ymm0, %ymm0
; AVX512DQ-NEXT: vcvtudq2ps %zmm0, %zmm0
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512DQ-NEXT: ret{{[l|q]}}
Expand Down Expand Up @@ -330,10 +328,9 @@ define <4 x double> @uitofp_v4i32_v4f64(<4 x i32> %x) #0 {
; AVX1-NEXT: vaddpd %ymm0, %ymm1, %ymm0
; AVX1-NEXT: ret{{[l|q]}}
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512F-LABEL: uitofp_v4i32_v4f64:
; AVX512F: # %bb.0:
; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512F-NEXT: vmovaps %xmm0, %xmm0
; AVX512F-NEXT: vcvtudq2pd %ymm0, %zmm0
; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512F-NEXT: ret{{[l|q]}}
Expand All @@ -343,10 +340,9 @@ define <4 x double> @uitofp_v4i32_v4f64(<4 x i32> %x) #0 {
; AVX512VL-NEXT: vcvtudq2pd %xmm0, %ymm0
; AVX512VL-NEXT: ret{{[l|q]}}
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: uitofp_v4i32_v4f64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512DQ-NEXT: vmovaps %xmm0, %xmm0
; AVX512DQ-NEXT: vcvtudq2pd %ymm0, %zmm0
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512DQ-NEXT: ret{{[l|q]}}
Expand Down Expand Up @@ -412,10 +408,9 @@ define <4 x double> @sitofp_v4i64_v4f64(<4 x i64> %x) #0 {
; AVX-64-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX-64-NEXT: retq
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: sitofp_v4i64_v4f64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512DQ-NEXT: vmovaps %ymm0, %ymm0
; AVX512DQ-NEXT: vcvtqq2pd %zmm0, %zmm0
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512DQ-NEXT: ret{{[l|q]}}
Expand Down Expand Up @@ -503,10 +498,9 @@ define <4 x double> @uitofp_v4i64_v4f64(<4 x i64> %x) #0 {
; AVX512VL-64-NEXT: vaddpd %ymm0, %ymm1, %ymm0
; AVX512VL-64-NEXT: retq
;
; FIXME: This is an unsafe behavior for strict FP
; AVX512DQ-LABEL: uitofp_v4i64_v4f64:
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; AVX512DQ-NEXT: vmovaps %ymm0, %ymm0
; AVX512DQ-NEXT: vcvtuqq2pd %zmm0, %zmm0
; AVX512DQ-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0
; AVX512DQ-NEXT: ret{{[l|q]}}
Expand Down

0 comments on commit f953882

Please sign in to comment.