Skip to content

Commit

Permalink
[CodeGen] Generate efficient assembly for freeze(poison) version of `…
Browse files Browse the repository at this point in the history 
…mm*_cast*` intel intrinsics

This patch makes the variants of `mm*_cast*` intel intrinsics that use `shufflevector(freeze(poison), ..)` emit efficient assembly.
(These intrinsics are planned to use `shufflevector(freeze(poison), ..)` after shufflevector's semantics update; relevant thread: D103874)

To do so, this patch

1. Updates `LowerAVXCONCAT_VECTORS` in X86ISelLowering.cpp to recognize `FREEZE(UNDEF)` operand of `CONCAT_VECTOR` in addition to `UNDEF`
2. Updates X86InstrVecCompiler.td to recognize `insert_subvector` of `FREEZE(UNDEF)` vector as its first operand.

Reviewed By: craig.topper

Differential Revision: https://reviews.llvm.org/D130339
  • Loading branch information
@aqjune
aqjune committed Aug 11, 2022
1 parent 5e0c3b4 commit 02e56e2
Show file tree
Hide file tree
Showing 9 changed files with 33 additions and 44 deletions.
3 changes: 3 additions & 0 deletions llvm/include/llvm/CodeGen/SelectionDAGNodes.h
View file
Expand Up @@ -122,6 +122,9 @@ bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
/// specified node are ISD::UNDEF.
bool allOperandsUndef(const SDNode *N);

/// Return true if the specified node is FREEZE(UNDEF).
bool isFreezeUndef(const SDNode *N);

} // end namespace ISD

//===----------------------------------------------------------------------===//
Expand Down
7 changes: 7 additions & 0 deletions llvm/include/llvm/Target/TargetSelectionDAG.td
View file
Expand Up @@ -175,6 +175,9 @@ def SDTExtInvec : SDTypeProfile<1, 1, [ // sext_invec
SDTCisInt<0>, SDTCisVec<0>, SDTCisInt<1>, SDTCisVec<1>,
SDTCisOpSmallerThanOp<1, 0>
]>;
def SDTFreeze : SDTypeProfile<1, 1, [
SDTCisSameAs<0, 1>
]>;

def SDTSetCC : SDTypeProfile<1, 3, [ // setcc
SDTCisInt<0>, SDTCisSameAs<1, 2>, SDTCisVT<3, OtherVT>
Expand Down Expand Up @@ -453,6 +456,7 @@ def anyext : SDNode<"ISD::ANY_EXTEND" , SDTIntExtendOp>;
def trunc : SDNode<"ISD::TRUNCATE" , SDTIntTruncOp>;
def bitconvert : SDNode<"ISD::BITCAST" , SDTUnaryOp>;
def addrspacecast : SDNode<"ISD::ADDRSPACECAST", SDTUnaryOp>;
def freeze : SDNode<"ISD::FREEZE" , SDTFreeze>;
def extractelt : SDNode<"ISD::EXTRACT_VECTOR_ELT", SDTVecExtract>;
def insertelt : SDNode<"ISD::INSERT_VECTOR_ELT", SDTVecInsert>;

Expand Down Expand Up @@ -1300,6 +1304,9 @@ def post_truncstvi16 : PatFrag<(ops node:$val, node:$base, node:$offset),
let ScalarMemoryVT = i16;
}

// A helper for matching undef or freeze undef
def undef_or_freeze_undef : PatFrags<(ops), [(undef), (freeze undef)]>;

// TODO: Split these into volatile and unordered flavors to enable
// selectively legal optimizations for each. (See D66309)
def simple_load : PatFrag<(ops node:$ptr),
Expand Down
4 changes: 4 additions & 0 deletions llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View file
Expand Up @@ -300,6 +300,10 @@ bool ISD::allOperandsUndef(const SDNode *N) {
return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
}

bool ISD::isFreezeUndef(const SDNode *N) {
return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();
}

bool ISD::matchUnaryPredicate(SDValue Op,
std::function<bool(ConstantSDNode *)> Match,
bool AllowUndefs) {
Expand Down
8 changes: 6 additions & 2 deletions llvm/lib/Target/X86/X86ISelLowering.cpp
View file
Expand Up @@ -11461,14 +11461,17 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide");

unsigned NumOperands = Op.getNumOperands();
unsigned NumFreezeUndef = 0;
unsigned NumZero = 0;
unsigned NumNonZero = 0;
unsigned NonZeros = 0;
for (unsigned i = 0; i != NumOperands; ++i) {
SDValue SubVec = Op.getOperand(i);
if (SubVec.isUndef())
continue;
if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
if (ISD::isFreezeUndef(SubVec.getNode()) && SubVec.hasOneUse())
++NumFreezeUndef;
else if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
++NumZero;
else {
assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
Expand All @@ -11490,7 +11493,8 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,

// Otherwise, build it up through insert_subvectors.
SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
: DAG.getUNDEF(ResVT);
: (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT))
: DAG.getUNDEF(ResVT));

MVT SubVT = Op.getOperand(0).getSimpleValueType();
unsigned NumSubElems = SubVT.getVectorNumElements();
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/X86/X86InstrVecCompiler.td
View file
Expand Up @@ -68,7 +68,7 @@ multiclass subvector_subreg_lowering<RegisterClass subRC, ValueType subVT,
def : Pat<(subVT (extract_subvector (VT RC:$src), (iPTR 0))),
(subVT (EXTRACT_SUBREG RC:$src, subIdx))>;

def : Pat<(VT (insert_subvector undef, subRC:$src, (iPTR 0))),
def : Pat<(VT (insert_subvector undef_or_freeze_undef, subRC:$src, (iPTR 0))),
(VT (INSERT_SUBREG (IMPLICIT_DEF), subRC:$src, subIdx))>;
}

Expand Down
3 changes: 0 additions & 3 deletions llvm/test/CodeGen/X86/avx-intrinsics-fast-isel.ll
View file
Expand Up @@ -258,7 +258,6 @@ define <4 x double> @test_mm256_castpd128_pd256_freeze(<2 x double> %a0) nounwin
; CHECK-LABEL: test_mm256_castpd128_pd256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%a1 = freeze <2 x double> poison
%res = shufflevector <2 x double> %a0, <2 x double> %a1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
Expand Down Expand Up @@ -304,7 +303,6 @@ define <8 x float> @test_mm256_castps128_ps256_freeze(<4 x float> %a0) nounwind
; CHECK-LABEL: test_mm256_castps128_ps256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%a1 = freeze <4 x float> poison
%res = shufflevector <4 x float> %a0, <4 x float> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
Expand Down Expand Up @@ -334,7 +332,6 @@ define <4 x i64> @test_mm256_castsi128_si256_freeze(<2 x i64> %a0) nounwind {
; CHECK-LABEL: test_mm256_castsi128_si256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%a1 = freeze <2 x i64> poison
%res = shufflevector <2 x i64> %a0, <2 x i64> %a1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
Expand Down
45 changes: 12 additions & 33 deletions llvm/test/CodeGen/X86/avx-intrinsics-x86.ll
View file
Expand Up @@ -1033,53 +1033,32 @@ declare <2 x i64> @llvm.x86.pclmulqdq(<2 x i64>, <2 x i64>, i8) nounwind readnon


define <4 x double> @test_mm256_castpd128_pd256_freeze(<2 x double> %a0) nounwind {
; AVX-LABEL: test_mm256_castpd128_pd256_freeze:
; AVX: # %bb.0:
; AVX-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x18,0xc0,0x01]
; AVX-NEXT: ret{{[l|q]}} # encoding: [0xc3]
;
; AVX512VL-LABEL: test_mm256_castpd128_pd256_freeze:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512VL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x18,0xc0,0x01]
; AVX512VL-NEXT: ret{{[l|q]}} # encoding: [0xc3]
; CHECK-LABEL: test_mm256_castpd128_pd256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%a1 = freeze <2 x double> poison
%res = shufflevector <2 x double> %a0, <2 x double> %a1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x double> %res
}


define <8 x float> @test_mm256_castps128_ps256_freeze(<4 x float> %a0) nounwind {
; AVX-LABEL: test_mm256_castps128_ps256_freeze:
; AVX: # %bb.0:
; AVX-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x18,0xc0,0x01]
; AVX-NEXT: ret{{[l|q]}} # encoding: [0xc3]
;
; AVX512VL-LABEL: test_mm256_castps128_ps256_freeze:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512VL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x18,0xc0,0x01]
; AVX512VL-NEXT: ret{{[l|q]}} # encoding: [0xc3]
; CHECK-LABEL: test_mm256_castps128_ps256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%a1 = freeze <4 x float> poison
%res = shufflevector <4 x float> %a0, <4 x float> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x float> %res
}


define <4 x i64> @test_mm256_castsi128_si256_freeze(<2 x i64> %a0) nounwind {
; AVX-LABEL: test_mm256_castsi128_si256_freeze:
; AVX: # %bb.0:
; AVX-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 # encoding: [0xc4,0xe3,0x7d,0x18,0xc0,0x01]
; AVX-NEXT: ret{{[l|q]}} # encoding: [0xc3]
;
; AVX512VL-LABEL: test_mm256_castsi128_si256_freeze:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX512VL-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe3,0x7d,0x18,0xc0,0x01]
; AVX512VL-NEXT: ret{{[l|q]}} # encoding: [0xc3]
; CHECK-LABEL: test_mm256_castsi128_si256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%a1 = freeze <2 x i64> poison
%res = shufflevector <2 x i64> %a0, <2 x i64> %a1, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x i64> %res
Expand Down
3 changes: 0 additions & 3 deletions llvm/test/CodeGen/X86/avx512-intrinsics.ll
View file
Expand Up @@ -7510,7 +7510,6 @@ define <8 x double> @test_mm256_castpd256_pd256_freeze(<4 x double> %a0) nounwin
; CHECK-LABEL: test_mm256_castpd256_pd256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; CHECK-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
; CHECK-NEXT: ret{{[l|q]}}
%a1 = freeze <4 x double> poison
%res = shufflevector <4 x double> %a0, <4 x double> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
Expand All @@ -7536,7 +7535,6 @@ define <16 x float> @test_mm256_castps256_ps512_freeze(<8 x float> %a0) nounwind
; CHECK-LABEL: test_mm256_castps256_ps512_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; CHECK-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
; CHECK-NEXT: ret{{[l|q]}}
%a1 = freeze <8 x float> poison
%res = shufflevector <8 x float> %a0, <8 x float> %a1, <16x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
Expand All @@ -7562,7 +7560,6 @@ define <8 x i64> @test_mm512_castsi256_si512_pd256_freeze(<4 x i64> %a0) nounwin
; CHECK-LABEL: test_mm512_castsi256_si512_pd256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; CHECK-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
; CHECK-NEXT: ret{{[l|q]}}
%a1 = freeze <4 x i64> poison
%res = shufflevector <4 x i64> %a0, <4 x i64> %a1, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
Expand Down
2 changes: 0 additions & 2 deletions llvm/test/CodeGen/X86/avx512fp16-intrinsics.ll
View file
Expand Up @@ -1221,7 +1221,6 @@ define <16 x half> @test_mm256_castph128_ph256_freeze(<8 x half> %a0) nounwind {
; CHECK-LABEL: test_mm256_castph128_ph256_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
; CHECK-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
; CHECK-NEXT: retq
%a1 = freeze <8 x half> poison
%res = shufflevector <8 x half> %a0, <8 x half> %a1, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
Expand All @@ -1247,7 +1246,6 @@ define <32 x half> @test_mm512_castph256_ph512_freeze(<16 x half> %a0) nounwind
; CHECK-LABEL: test_mm512_castph256_ph512_freeze:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
; CHECK-NEXT: vinsertf64x4 $1, %ymm0, %zmm0, %zmm0
; CHECK-NEXT: retq
%a1 = freeze <16 x half> poison
%res = shufflevector <16 x half> %a0, <16 x half> %a1, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
Expand Down

0 comments on commit 02e56e2

Please sign in to comment.