diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 9a6dd7735421e..bb06308132442 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -3324,6 +3324,12 @@ class LLVM_ABI TargetLoweringBase { /// Default to be the minimum interleave factor: 2. virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; } + /// Return true if target supports interleave intrinsics. + virtual bool isInterleaveIntrinsicSupported(unsigned /*Factor*/, + EVT VT) const { + return VT.isScalableVector(); + } + /// Lower an interleaved load to target specific intrinsics. Return /// true on success. /// diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index d4d56a9563f71..5a7df7c1b73c6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -1079,6 +1079,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer { SDValue WidenVecRes_VECTOR_REVERSE(SDNode *N); SDValue WidenVecRes_GET_ACTIVE_LANE_MASK(SDNode *N); void WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N); + void WidenVecRes_VECTOR_INTERLEAVE(SDNode *N); SDValue WidenVecRes_Ternary(SDNode *N); SDValue WidenVecRes_Binary(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index 3ac1ca367abc8..e89cb03cdd4ab 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -5331,6 +5331,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { case ISD::VECTOR_DEINTERLEAVE: WidenVecRes_VECTOR_DEINTERLEAVE(N); break; + case ISD::VECTOR_INTERLEAVE: + WidenVecRes_VECTOR_INTERLEAVE(N); + break; case ISD::ADD: case ISD::VP_ADD: case ISD::AND: case ISD::VP_AND: @@ -7523,6 +7526,41 @@ void DAGTypeLegalizer::WidenVecRes_VECTOR_DEINTERLEAVE(SDNode *N) { SetWidenedVector(SDValue(N, Idx), NewRes.getValue(Idx)); } +void DAGTypeLegalizer::WidenVecRes_VECTOR_INTERLEAVE(SDNode *N) { + EVT VT = N->getValueType(0); + EVT EltVT = VT.getVectorElementType(); + ElementCount OrigEC = VT.getVectorElementCount(); + unsigned Factor = N->getNumOperands(); + SDLoc DL(N); + + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); + ElementCount WidenEC = WidenVT.getVectorElementCount(); + + SmallVector WidenOps(Factor); + for (unsigned Idx = 0U; Idx < Factor; ++Idx) + WidenOps[Idx] = GetWidenedVector(N->getOperand(Idx)); + + SmallVector WidenVTs(Factor, WidenVT); + SDValue Interleaved = + DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, WidenVTs, WidenOps); + + EVT PackedWidenVT = EVT::getVectorVT(*DAG.getContext(), EltVT, + WidenEC.multiplyCoefficientBy(Factor)); + SmallVector Slices(Factor); + for (unsigned Idx = 0; Idx != Factor; ++Idx) + Slices[Idx] = Interleaved.getValue(Idx); + + SDValue Packed = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedWidenVT, Slices); + + for (unsigned Idx = 0U; Idx < Factor; ++Idx) { + SDValue Narrow = DAG.getExtractSubvector( + DL, VT, Packed, OrigEC.multiplyCoefficientBy(Idx).getKnownMinValue()); + SDValue Wide = + DAG.getInsertSubvector(DL, DAG.getUNDEF(WidenVT), Narrow, /*Idx=*/0U); + SetWidenedVector(SDValue(N, Idx), Wide); + } +} + SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { assert(N->getValueType(0).isVector() && N->getOperand(0).getValueType().isVector() && diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 0f6bd53cafdd8..60fd7e93eaae2 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -13049,34 +13049,44 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I, unsigned Factor) { auto DL = getCurSDLoc(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue InVec = getValue(I.getOperand(0)); + EVT InVT = InVec.getValueType(); SmallVector ValueVTs; - ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), - ValueVTs); + ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); EVT OutVT = ValueVTs[0]; unsigned OutNumElts = OutVT.getVectorMinNumElements(); - SmallVector SubVecs(Factor); - for (unsigned i = 0; i != Factor; ++i) { - assert(ValueVTs[i] == OutVT && "Expected VTs to be the same"); - SubVecs[i] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, - DAG.getVectorIdxConstant(OutNumElts * i, DL)); - } - - // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit - // from existing legalisation and combines. - if (OutVT.isFixedLengthVector() && Factor == 2) { - SDValue Even = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1], - createStrideMask(0, 2, OutNumElts)); - SDValue Odd = DAG.getVectorShuffle(OutVT, DL, SubVecs[0], SubVecs[1], - createStrideMask(1, 2, OutNumElts)); - SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc()); - setValue(&I, Res); + // Use VECTOR_SHUFFLE for fixed-length vectors that the target does not lower + // natively. + if (!TLI.isInterleaveIntrinsicSupported(Factor, InVT)) { + unsigned WideNumElts = InVT.getVectorNumElements(); + + SmallVector Results; + Results.reserve(Factor); + for (unsigned I = 0; I != Factor; ++I) { + SmallVector Mask(WideNumElts, -1); + for (unsigned J = 0; J != OutNumElts; ++J) + Mask[J] = I + J * Factor; + SDValue Shuffle = + DAG.getVectorShuffle(InVT, DL, InVec, DAG.getUNDEF(InVT), Mask); + Results.push_back(DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, Shuffle, + DAG.getVectorIdxConstant(0, DL))); + } + + setValue(&I, DAG.getMergeValues(Results, DL)); return; } + SmallVector SubVecs(Factor); + for (unsigned I = 0; I != Factor; ++I) { + assert(ValueVTs[I] == OutVT && "Expected VTs to be the same"); + SubVecs[I] = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, + DAG.getVectorIdxConstant(OutNumElts * I, DL)); + } + SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, DAG.getVTList(ValueVTs), SubVecs); setValue(&I, Res); @@ -13096,13 +13106,13 @@ void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I, "Expected VTs to be the same"); } - // Use VECTOR_SHUFFLE for fixed-length vectors with factor of 2 to benefit - // from existing legalisation and combines. - if (OutVT.isFixedLengthVector() && Factor == 2) { + // Use VECTOR_SHUFFLE for fixed-length vectors that the target does not lower + // natively. + if (!TLI.isInterleaveIntrinsicSupported(Factor, OutVT)) { unsigned NumElts = InVT.getVectorMinNumElements(); SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVecs); setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT), - createInterleaveMask(NumElts, 2))); + createInterleaveMask(NumElts, Factor))); return; } diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 6e395e004f519..8caffe89f1b4d 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -232,6 +232,10 @@ class AArch64TargetLowering : public TargetLowering { unsigned getMaxSupportedInterleaveFactor() const override { return 4; } + bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override { + return VT.isScalableVector() || Factor == 3 || Factor % 2 == 0; + } + bool lowerInterleavedLoad(Instruction *Load, Value *Mask, ArrayRef Shuffles, ArrayRef Indices, unsigned Factor, diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index f8a99d38e2691..82430c6241ef7 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -437,6 +437,10 @@ class RISCVTargetLowering : public TargetLowering { unsigned getMaxSupportedInterleaveFactor() const override { return 8; } + bool isInterleaveIntrinsicSupported(unsigned Factor, EVT VT) const override { + return VT.isScalableVector() || Factor > 2; + } + bool fallBackToDAGISel(const Instruction &Inst) const override; bool lowerInterleavedLoad(Instruction *Load, Value *Mask, diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll index 355adec955e4b..3e3e3389c146d 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-reductions.ll @@ -31,8 +31,8 @@ define dso_local %"struct.std::complex" @complex_mul_v2f64(ptr %a, ptr %b) { ; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90 ; CHECK-NEXT: b.ne .LBB0_1 ; CHECK-NEXT: // %bb.2: // %middle.block -; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d -; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d +; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d +; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d ; CHECK-NEXT: faddp d0, v0.2d ; CHECK-NEXT: faddp d1, v2.2d ; CHECK-NEXT: ret @@ -80,10 +80,13 @@ middle.block: ; preds = %vector.body define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_nonzero_init_v2f64: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: movi v0.2d, #0000000000000000 ; CHECK-NEXT: adrp x8, .LCPI1_0 +; CHECK-NEXT: adrp x9, .LCPI1_1 ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI1_0] +; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI1_1] ; CHECK-NEXT: mov x8, xzr +; CHECK-NEXT: zip2 v0.2d, v2.2d, v1.2d +; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d ; CHECK-NEXT: .LBB1_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: add x9, x0, x8 @@ -98,8 +101,8 @@ define %"struct.std::complex" @complex_mul_nonzero_init_v2f64(ptr %a, ptr %b) { ; CHECK-NEXT: fcmla v0.2d, v4.2d, v2.2d, #90 ; CHECK-NEXT: b.ne .LBB1_1 ; CHECK-NEXT: // %bb.2: // %middle.block -; CHECK-NEXT: zip2 v2.2d, v1.2d, v0.2d -; CHECK-NEXT: zip1 v0.2d, v1.2d, v0.2d +; CHECK-NEXT: uzp2 v2.2d, v1.2d, v0.2d +; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d ; CHECK-NEXT: faddp d0, v0.2d ; CHECK-NEXT: faddp d1, v2.2d ; CHECK-NEXT: ret @@ -143,14 +146,17 @@ middle.block: ; preds = %vector.body define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-LABEL: complex_mul_v2f64_unrolled: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: adrp x9, .LCPI2_1 ; CHECK-NEXT: movi v3.2d, #0000000000000000 -; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI2_0] +; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI2_0] +; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI2_1] ; CHECK-NEXT: add x8, x0, #32 ; CHECK-NEXT: add x9, x1, #32 ; CHECK-NEXT: mov x10, #-100 // =0xffffffffffffff9c +; CHECK-NEXT: zip2 v0.2d, v2.2d, v1.2d +; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d +; CHECK-NEXT: movi v2.2d, #0000000000000000 ; CHECK-NEXT: .LBB2_1: // %vector.body ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1 ; CHECK-NEXT: ldp q5, q4, [x8, #-32] @@ -158,24 +164,24 @@ define %"struct.std::complex" @complex_mul_v2f64_unrolled(ptr %a, ptr %b) { ; CHECK-NEXT: ldp q7, q6, [x9, #-32] ; CHECK-NEXT: ldp q17, q16, [x8], #64 ; CHECK-NEXT: ldp q19, q18, [x9], #64 -; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #0 -; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #0 -; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #0 -; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #0 -; CHECK-NEXT: fcmla v2.2d, v7.2d, v5.2d, #90 -; CHECK-NEXT: fcmla v1.2d, v6.2d, v4.2d, #90 -; CHECK-NEXT: fcmla v0.2d, v19.2d, v17.2d, #90 -; CHECK-NEXT: fcmla v3.2d, v18.2d, v16.2d, #90 +; CHECK-NEXT: fcmla v1.2d, v7.2d, v5.2d, #0 +; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #0 +; CHECK-NEXT: fcmla v3.2d, v19.2d, v17.2d, #0 +; CHECK-NEXT: fcmla v2.2d, v18.2d, v16.2d, #0 +; CHECK-NEXT: fcmla v1.2d, v7.2d, v5.2d, #90 +; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #90 +; CHECK-NEXT: fcmla v3.2d, v19.2d, v17.2d, #90 +; CHECK-NEXT: fcmla v2.2d, v18.2d, v16.2d, #90 ; CHECK-NEXT: b.ne .LBB2_1 ; CHECK-NEXT: // %bb.2: // %middle.block -; CHECK-NEXT: zip2 v4.2d, v0.2d, v3.2d -; CHECK-NEXT: zip1 v0.2d, v0.2d, v3.2d -; CHECK-NEXT: zip2 v3.2d, v2.2d, v1.2d -; CHECK-NEXT: zip1 v1.2d, v2.2d, v1.2d -; CHECK-NEXT: fadd v0.2d, v0.2d, v1.2d +; CHECK-NEXT: uzp2 v4.2d, v3.2d, v2.2d +; CHECK-NEXT: uzp1 v2.2d, v3.2d, v2.2d +; CHECK-NEXT: uzp2 v3.2d, v1.2d, v0.2d +; CHECK-NEXT: uzp1 v0.2d, v1.2d, v0.2d ; CHECK-NEXT: fadd v1.2d, v4.2d, v3.2d -; CHECK-NEXT: faddp d0, v0.2d +; CHECK-NEXT: fadd v0.2d, v2.2d, v0.2d ; CHECK-NEXT: faddp d1, v1.2d +; CHECK-NEXT: faddp d0, v0.2d ; CHECK-NEXT: ret entry: %scevgep = getelementptr i8, ptr %a, i64 32 diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll index ad9240b0922bd..f8c7a25f47185 100644 --- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll +++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-splat.ll @@ -9,21 +9,23 @@ target triple = "aarch64" define <4 x double> @complex_mul_const(<4 x double> %a, <4 x double> %b) { ; CHECK-LABEL: complex_mul_const: ; CHECK: // %bb.0: // %entry -; CHECK-NEXT: movi v6.2d, #0000000000000000 -; CHECK-NEXT: movi v5.2d, #0000000000000000 -; CHECK-NEXT: adrp x8, .LCPI0_0 ; CHECK-NEXT: movi v4.2d, #0000000000000000 -; CHECK-NEXT: fcmla v6.2d, v1.2d, v3.2d, #0 -; CHECK-NEXT: fcmla v5.2d, v0.2d, v2.2d, #0 -; CHECK-NEXT: fcmla v6.2d, v1.2d, v3.2d, #90 -; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0] -; CHECK-NEXT: fcmla v5.2d, v0.2d, v2.2d, #90 +; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: fmov v6.2d, #3.00000000 +; CHECK-NEXT: fmov v7.2d, #11.00000000 +; CHECK-NEXT: fcmla v4.2d, v0.2d, v2.2d, #0 +; CHECK-NEXT: fcmla v5.2d, v1.2d, v3.2d, #0 +; CHECK-NEXT: fcmla v4.2d, v0.2d, v2.2d, #90 +; CHECK-NEXT: movi v2.2d, #0000000000000000 +; CHECK-NEXT: fcmla v5.2d, v1.2d, v3.2d, #90 +; CHECK-NEXT: zip2 v1.2d, v7.2d, v6.2d +; CHECK-NEXT: zip1 v3.2d, v7.2d, v6.2d ; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: fcmla v4.2d, v6.2d, v1.2d, #0 -; CHECK-NEXT: fcmla v0.2d, v5.2d, v1.2d, #0 -; CHECK-NEXT: fcmla v4.2d, v6.2d, v1.2d, #90 -; CHECK-NEXT: fcmla v0.2d, v5.2d, v1.2d, #90 -; CHECK-NEXT: mov v1.16b, v4.16b +; CHECK-NEXT: fcmla v2.2d, v5.2d, v1.2d, #0 +; CHECK-NEXT: fcmla v0.2d, v4.2d, v3.2d, #0 +; CHECK-NEXT: fcmla v2.2d, v5.2d, v1.2d, #90 +; CHECK-NEXT: fcmla v0.2d, v4.2d, v3.2d, #90 +; CHECK-NEXT: mov v1.16b, v2.16b ; CHECK-NEXT: ret entry: %strided.vec = shufflevector <4 x double> %a, <4 x double> poison, <2 x i32> @@ -56,18 +58,21 @@ define <4 x double> @complex_mul_non_const(<4 x double> %a, <4 x double> %b, [2 ; CHECK-NEXT: movi v6.2d, #0000000000000000 ; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5 ; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4 -; CHECK-NEXT: mov v4.d[1], v5.d[0] -; CHECK-NEXT: movi v5.2d, #0000000000000000 +; CHECK-NEXT: dup v5.2d, v5.d[0] +; CHECK-NEXT: dup v4.2d, v4.d[0] ; CHECK-NEXT: fcmla v7.2d, v1.2d, v3.2d, #0 ; CHECK-NEXT: fcmla v6.2d, v0.2d, v2.2d, #0 +; CHECK-NEXT: zip2 v16.2d, v4.2d, v5.2d +; CHECK-NEXT: zip1 v5.2d, v4.2d, v5.2d +; CHECK-NEXT: movi v4.2d, #0000000000000000 ; CHECK-NEXT: fcmla v7.2d, v1.2d, v3.2d, #90 ; CHECK-NEXT: fcmla v6.2d, v0.2d, v2.2d, #90 ; CHECK-NEXT: movi v0.2d, #0000000000000000 -; CHECK-NEXT: fcmla v5.2d, v7.2d, v4.2d, #0 -; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #0 -; CHECK-NEXT: fcmla v5.2d, v7.2d, v4.2d, #90 -; CHECK-NEXT: fcmla v0.2d, v6.2d, v4.2d, #90 -; CHECK-NEXT: mov v1.16b, v5.16b +; CHECK-NEXT: fcmla v4.2d, v7.2d, v16.2d, #0 +; CHECK-NEXT: fcmla v0.2d, v6.2d, v5.2d, #0 +; CHECK-NEXT: fcmla v4.2d, v7.2d, v16.2d, #90 +; CHECK-NEXT: fcmla v0.2d, v6.2d, v5.2d, #90 +; CHECK-NEXT: mov v1.16b, v4.16b ; CHECK-NEXT: ret entry: %c.coerce.fca.1.extract = extractvalue [2 x double] %c, 1 diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll index 8c48cc609c75a..2420164619be0 100644 --- a/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll +++ b/llvm/test/CodeGen/AArch64/fixed-vector-deinterleave.ll @@ -3,21 +3,12 @@ ; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel -global-isel-abort=0 %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define {<2 x half>, <2 x half>} @vector_deinterleave_v2f16_v4f16(<4 x half> %vec) { -; CHECK-SD-LABEL: vector_deinterleave_v2f16_v4f16: -; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: dup v1.2s, v0.s[1] -; CHECK-SD-NEXT: zip1 v2.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: trn2 v1.4h, v0.4h, v1.4h -; CHECK-SD-NEXT: fmov d0, d2 -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: vector_deinterleave_v2f16_v4f16: -; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: uzp1 v2.4h, v0.4h, v0.4h -; CHECK-GI-NEXT: uzp2 v1.4h, v0.4h, v0.4h -; CHECK-GI-NEXT: fmov d0, d2 -; CHECK-GI-NEXT: ret +; CHECK-LABEL: vector_deinterleave_v2f16_v4f16: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v2.4h, v0.4h, v0.4h +; CHECK-NEXT: uzp2 v1.4h, v0.4h, v0.4h +; CHECK-NEXT: fmov d0, d2 +; CHECK-NEXT: ret %retval = call {<2 x half>, <2 x half>} @llvm.vector.deinterleave2.v4f16(<4 x half> %vec) ret {<2 x half>, <2 x half>} %retval } @@ -49,8 +40,8 @@ define {<2 x float>, <2 x float>} @vector_deinterleave_v2f32_v4f32(<4 x float> % ; CHECK-SD-LABEL: vector_deinterleave_v2f32_v4f32: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: mov d1, v0.d[1] -; CHECK-SD-NEXT: zip1 v2.2s, v0.2s, v1.2s -; CHECK-SD-NEXT: zip2 v1.2s, v0.2s, v1.2s +; CHECK-SD-NEXT: uzp1 v2.2s, v0.2s, v1.2s +; CHECK-SD-NEXT: uzp2 v1.2s, v0.2s, v1.2s ; CHECK-SD-NEXT: fmov d0, d2 ; CHECK-SD-NEXT: ret ; @@ -77,12 +68,19 @@ ret {<4 x float>, <4 x float>} %retval } define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double> %vec) { -; CHECK-LABEL: vector_deinterleave_v2f64_v4f64: -; CHECK: // %bb.0: -; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d -; CHECK-NEXT: zip2 v1.2d, v0.2d, v1.2d -; CHECK-NEXT: mov v0.16b, v2.16b -; CHECK-NEXT: ret +; CHECK-SD-LABEL: vector_deinterleave_v2f64_v4f64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: mov v0.16b, v2.16b +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: vector_deinterleave_v2f64_v4f64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d +; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d +; CHECK-GI-NEXT: mov v0.16b, v2.16b +; CHECK-GI-NEXT: ret %retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec) ret {<2 x double>, <2 x double>} %retval } @@ -123,12 +121,19 @@ define {<4 x i32>, <4 x i32>} @vector_deinterleave_v4i32_v8i32(<8 x i32> %vec) { } define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) { -; CHECK-LABEL: vector_deinterleave_v2i64_v4i64: -; CHECK: // %bb.0: -; CHECK-NEXT: zip1 v2.2d, v0.2d, v1.2d -; CHECK-NEXT: zip2 v1.2d, v0.2d, v1.2d -; CHECK-NEXT: mov v0.16b, v2.16b -; CHECK-NEXT: ret +; CHECK-SD-LABEL: vector_deinterleave_v2i64_v4i64: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: uzp1 v2.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: uzp2 v1.2d, v0.2d, v1.2d +; CHECK-SD-NEXT: mov v0.16b, v2.16b +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: vector_deinterleave_v2i64_v4i64: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: zip1 v2.2d, v0.2d, v1.2d +; CHECK-GI-NEXT: zip2 v1.2d, v0.2d, v1.2d +; CHECK-GI-NEXT: mov v0.16b, v2.16b +; CHECK-GI-NEXT: ret %retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec) ret {<2 x i64>, <2 x i64>} %retval } @@ -846,3 +851,102 @@ define {<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 %retval = call {<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>} @llvm.vector.deinterleave8.v64bf16(<64 x bfloat> %ins7) ret {<8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>, <8 x bfloat>} %retval } + +define {<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>} @vector_deinterleave5_v20i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c, <4 x i16> %d, <4 x i16> %e) { +; CHECK-LABEL: vector_deinterleave5_v20i16: +; CHECK: // %bb.0: +; CHECK-NEXT: fmov d6, d2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3 +; CHECK-NEXT: adrp x8, .LCPI42_1 +; CHECK-NEXT: adrp x9, .LCPI42_2 +; CHECK-NEXT: fmov d5, d0 +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI42_1] +; CHECK-NEXT: adrp x8, .LCPI42_4 +; CHECK-NEXT: ldr q7, [x8, :lo12:.LCPI42_4] +; CHECK-NEXT: adrp x10, .LCPI42_3 +; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI42_2] +; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4 +; CHECK-NEXT: adrp x8, .LCPI42_0 +; CHECK-NEXT: mov v5.d[1], v1.d[0] +; CHECK-NEXT: mov v6.d[1], v3.d[0] +; CHECK-NEXT: ldr q3, [x10, :lo12:.LCPI42_3] +; CHECK-NEXT: tbl v7.16b, { v5.16b, v6.16b }, v7.16b +; CHECK-NEXT: tbl v1.16b, { v5.16b, v6.16b }, v0.16b +; CHECK-NEXT: tbl v2.16b, { v5.16b, v6.16b }, v2.16b +; CHECK-NEXT: tbl v3.16b, { v5.16b, v6.16b }, v3.16b +; CHECK-NEXT: ldr q0, [x8, :lo12:.LCPI42_0] +; CHECK-NEXT: tbl v0.16b, { v5.16b, v6.16b }, v0.16b +; CHECK-NEXT: mov v7.h[3], v4.h[3] +; CHECK-NEXT: mov v1.h[3], v4.h[0] +; CHECK-NEXT: mov v2.h[3], v4.h[1] +; CHECK-NEXT: mov v3.h[3], v4.h[2] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1 +; CHECK-NEXT: // kill: def $d2 killed $d2 killed $q2 +; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3 +; CHECK-NEXT: fmov d4, d7 +; CHECK-NEXT: ret + %ins0 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> poison, <4 x i16> %a, i64 0) + %ins1 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins0, <4 x i16> %b, i64 4) + %ins2 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins1, <4 x i16> %c, i64 8) + %ins3 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins2, <4 x i16> %d, i64 12) + %ins4 = call <20 x i16> @llvm.vector.insert.v20i16.v4i16(<20 x i16> %ins3, <4 x i16> %e, i64 16) + %retval = call {<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>} @llvm.vector.deinterleave5.v20i16(<20 x i16> %ins4) + ret {<4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>, <4 x i16>} %retval +} + +define {<4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>} @vector_deinterleave7_v28i8(<4 x i8> %a, <4 x i8> %b, <4 x i8> %c, <4 x i8> %d, <4 x i8> %e, <4 x i8> %f, <4 x i8> %g) { +; CHECK-LABEL: vector_deinterleave7_v28i8: +; CHECK: // %bb.0: +; CHECK-NEXT: uzp1 v19.8b, v0.8b, v1.8b +; CHECK-NEXT: uzp1 v18.8b, v2.8b, v3.8b +; CHECK-NEXT: // kill: def $d6 killed $d6 def $q6 +; CHECK-NEXT: uzp1 v4.8b, v4.8b, v5.8b +; CHECK-NEXT: xtn v5.8b, v6.8h +; CHECK-NEXT: mov b7, v19.b[4] +; CHECK-NEXT: mov b16, v19.b[5] +; CHECK-NEXT: mov b17, v19.b[6] +; CHECK-NEXT: mov b0, v19.b[0] +; CHECK-NEXT: mov b1, v19.b[1] +; CHECK-NEXT: mov b2, v19.b[2] +; CHECK-NEXT: mov b3, v19.b[3] +; CHECK-NEXT: mov v7.b[2], v18.b[3] +; CHECK-NEXT: mov v16.b[2], v18.b[4] +; CHECK-NEXT: mov v17.b[2], v18.b[5] +; CHECK-NEXT: mov v0.b[2], v19.b[7] +; CHECK-NEXT: mov v1.b[2], v18.b[0] +; CHECK-NEXT: mov v2.b[2], v18.b[1] +; CHECK-NEXT: mov v3.b[2], v18.b[2] +; CHECK-NEXT: mov v7.b[4], v4.b[2] +; CHECK-NEXT: mov v16.b[4], v4.b[3] +; CHECK-NEXT: mov v17.b[4], v4.b[4] +; CHECK-NEXT: mov v0.b[4], v18.b[6] +; CHECK-NEXT: mov v1.b[4], v18.b[7] +; CHECK-NEXT: mov v2.b[4], v4.b[0] +; CHECK-NEXT: mov v3.b[4], v4.b[1] +; CHECK-NEXT: mov v7.b[6], v5.b[1] +; CHECK-NEXT: mov v16.b[6], v5.b[2] +; CHECK-NEXT: mov v17.b[6], v5.b[3] +; CHECK-NEXT: mov v0.b[6], v4.b[5] +; CHECK-NEXT: mov v1.b[6], v4.b[6] +; CHECK-NEXT: mov v2.b[6], v4.b[7] +; CHECK-NEXT: mov v3.b[6], v5.b[0] +; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1 +; CHECK-NEXT: // kill: def $d2 killed $d2 killed $q2 +; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3 +; CHECK-NEXT: fmov d4, d7 +; CHECK-NEXT: fmov d5, d16 +; CHECK-NEXT: fmov d6, d17 +; CHECK-NEXT: ret + %ins0 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> poison, <4 x i8> %a, i64 0) + %ins1 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins0, <4 x i8> %b, i64 4) + %ins2 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins1, <4 x i8> %c, i64 8) + %ins3 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins2, <4 x i8> %d, i64 12) + %ins4 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins3, <4 x i8> %e, i64 16) + %ins5 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins4, <4 x i8> %f, i64 20) + %ins6 = call <28 x i8> @llvm.vector.insert.v28i8.v4i8(<28 x i8> %ins5, <4 x i8> %g, i64 24) + %retval = call {<4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>} @llvm.vector.deinterleave7.v28i8(<28 x i8> %ins6) + ret {<4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>, <4 x i8>} %retval +} diff --git a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll index 2d368c0ec456f..cc67c054f89c4 100644 --- a/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll +++ b/llvm/test/CodeGen/AArch64/fixed-vector-interleave.ll @@ -12,22 +12,12 @@ define <4 x half> @interleave2_v4f16(<2 x half> %vec0, <2 x half> %vec1) { } define <8 x half> @interleave2_v8f16(<4 x half> %vec0, <4 x half> %vec1) { -; CHECK-SD-LABEL: interleave2_v8f16: -; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: adrp x8, .LCPI1_0 -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI1_0] -; CHECK-SD-NEXT: tbl v0.16b, { v0.16b }, v1.16b -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: interleave2_v8f16: -; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-GI-NEXT: zip1 v0.8h, v0.8h, v1.8h -; CHECK-GI-NEXT: ret +; CHECK-LABEL: interleave2_v8f16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: zip1 v0.8h, v0.8h, v1.8h +; CHECK-NEXT: ret %retval = call <8 x half> @llvm.vector.interleave2.v8f16(<4 x half> %vec0, <4 x half> %vec1) ret <8 x half> %retval } @@ -44,21 +34,12 @@ define <16 x half> @interleave2_v16f16(<8 x half> %vec0, <8 x half> %vec1) { } define <4 x float> @interleave2_v4f32(<2 x float> %vec0, <2 x float> %vec1) { -; CHECK-SD-LABEL: interleave2_v4f32: -; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-SD-NEXT: mov v0.d[1], v1.d[0] -; CHECK-SD-NEXT: rev64 v1.4s, v0.4s -; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s -; CHECK-SD-NEXT: ret -; -; CHECK-GI-LABEL: interleave2_v4f32: -; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: // kill: def $d0 killed $d0 def $q0 -; CHECK-GI-NEXT: // kill: def $d1 killed $d1 def $q1 -; CHECK-GI-NEXT: zip1 v0.4s, v0.4s, v1.4s -; CHECK-GI-NEXT: ret +; CHECK-LABEL: interleave2_v4f32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: zip1 v0.4s, v0.4s, v1.4s +; CHECK-NEXT: ret %retval = call <4 x float> @llvm.vector.interleave2.v4f32(<2 x float> %vec0, <2 x float> %vec1) ret <4 x float> %retval } @@ -149,9 +130,10 @@ define <4 x i16> @interleave2_same_const_splat_v4i16() { define <4 x i16> @interleave2_diff_const_splat_v4i16() { ; CHECK-SD-LABEL: interleave2_diff_const_splat_v4i16: ; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: mov x8, #1125899907104768 // =0x4000000040000 -; CHECK-SD-NEXT: orr x8, x8, #0x300000003 -; CHECK-SD-NEXT: fmov d0, x8 +; CHECK-SD-NEXT: movi v0.2s, #4 +; CHECK-SD-NEXT: movi v1.2s, #3 +; CHECK-SD-NEXT: zip1 v0.4s, v1.4s, v0.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: interleave2_diff_const_splat_v4i16: @@ -184,12 +166,10 @@ define <4 x i16> @interleave2_same_nonconst_splat_v4i16(i16 %a) { define <4 x i16> @interleave2_diff_nonconst_splat_v4i16(i16 %a, i16 %b) { ; CHECK-SD-LABEL: interleave2_diff_nonconst_splat_v4i16: ; CHECK-SD: // %bb.0: -; CHECK-SD-NEXT: fmov s0, w0 -; CHECK-SD-NEXT: mov v0.h[1], w0 -; CHECK-SD-NEXT: mov v0.h[2], w1 -; CHECK-SD-NEXT: mov v0.h[3], w1 -; CHECK-SD-NEXT: rev32 v1.4h, v0.4h -; CHECK-SD-NEXT: uzp1 v0.4h, v0.4h, v1.4h +; CHECK-SD-NEXT: dup v0.2s, w0 +; CHECK-SD-NEXT: dup v1.2s, w1 +; CHECK-SD-NEXT: zip1 v0.4s, v0.4s, v1.4s +; CHECK-SD-NEXT: xtn v0.4h, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: interleave2_diff_nonconst_splat_v4i16: @@ -813,3 +793,65 @@ define <32 x i16> @interleave8_v32i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16 %retval = call <32 x i16> @llvm.vector.interleave8.v32i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3, <4 x i16> %vec4, <4 x i16> %vec5, <4 x i16> %vec6, <4 x i16> %vec7) ret <32 x i16> %retval } + +define <20 x i16> @interleave5_v20i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3, <4 x i16> %vec4) { +; CHECK-LABEL: interleave5_v20i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d4 killed $d4 killed $q3_q4 def $q3_q4 +; CHECK-NEXT: fmov d5, d3 +; CHECK-NEXT: fmov d7, d2 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: adrp x9, .LCPI45_0 +; CHECK-NEXT: fmov d6, d0 +; CHECK-NEXT: adrp x10, .LCPI45_2 +; CHECK-NEXT: ldr q0, [x9, :lo12:.LCPI45_0] +; CHECK-NEXT: adrp x9, .LCPI45_3 +; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI45_3] +; CHECK-NEXT: adrp x9, .LCPI45_1 +; CHECK-NEXT: mov v7.d[1], v5.d[0] +; CHECK-NEXT: mov v6.d[1], v1.d[0] +; CHECK-NEXT: ldr q1, [x10, :lo12:.LCPI45_2] +; CHECK-NEXT: tbl v3.16b, { v6.16b, v7.16b }, v0.16b +; CHECK-NEXT: tbl v0.16b, { v6.16b, v7.16b }, v1.16b +; CHECK-NEXT: tbl v1.16b, { v6.16b, v7.16b }, v2.16b +; CHECK-NEXT: ldr q2, [x9, :lo12:.LCPI45_1] +; CHECK-NEXT: mov v0.h[3], v4.h[3] +; CHECK-NEXT: mov v1.h[4], v4.h[0] +; CHECK-NEXT: tbl v2.16b, { v3.16b, v4.16b }, v2.16b +; CHECK-NEXT: str d0, [x8, #32] +; CHECK-NEXT: stp q1, q2, [x8] +; CHECK-NEXT: ret + %retval = call <20 x i16> @llvm.vector.interleave5.v20i16(<4 x i16> %vec0, <4 x i16> %vec1, <4 x i16> %vec2, <4 x i16> %vec3, <4 x i16> %vec4) + ret <20 x i16> %retval +} + +define <28 x i8> @interleave7_v28i8(<4 x i8> %vec0, <4 x i8> %vec1, <4 x i8> %vec2, <4 x i8> %vec3, <4 x i8> %vec4, <4 x i8> %vec5, <4 x i8> %vec6) { +; CHECK-LABEL: interleave7_v28i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2 +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 +; CHECK-NEXT: // kill: def $d6 killed $d6 killed $q4_q5_q6 def $q4_q5_q6 +; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3 +; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1 +; CHECK-NEXT: adrp x9, .LCPI46_0 +; CHECK-NEXT: mov v2.d[1], v3.d[0] +; CHECK-NEXT: mov v0.d[1], v1.d[0] +; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q4_q5_q6 def $q4_q5_q6 +; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI46_0] +; CHECK-NEXT: // kill: def $d4 killed $d4 killed $q4_q5_q6 def $q4_q5_q6 +; CHECK-NEXT: adrp x9, .LCPI46_1 +; CHECK-NEXT: tbl v4.16b, { v4.16b, v5.16b, v6.16b }, v1.16b +; CHECK-NEXT: uzp1 v3.16b, v0.16b, v2.16b +; CHECK-NEXT: ldr q0, [x9, :lo12:.LCPI46_1] +; CHECK-NEXT: adrp x9, .LCPI46_2 +; CHECK-NEXT: ldr q1, [x9, :lo12:.LCPI46_2] +; CHECK-NEXT: tbl v0.16b, { v3.16b, v4.16b }, v0.16b +; CHECK-NEXT: tbl v1.16b, { v3.16b, v4.16b }, v1.16b +; CHECK-NEXT: mov s2, v0.s[2] +; CHECK-NEXT: str q1, [x8] +; CHECK-NEXT: str d0, [x8, #16] +; CHECK-NEXT: str s2, [x8, #24] +; CHECK-NEXT: ret + %retval = call <28 x i8> @llvm.vector.interleave7.v28i8(<4 x i8> %vec0, <4 x i8> %vec1, <4 x i8> %vec2, <4 x i8> %vec3, <4 x i8> %vec4, <4 x i8> %vec5, <4 x i8> %vec6) + ret <28 x i8> %retval +} diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll index da917119c6b25..d8ff031239b41 100644 --- a/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll +++ b/llvm/test/CodeGen/AArch64/fixed_masked_deinterleaved_loads.ll @@ -5,25 +5,17 @@ define { <16 x i8>, <16 x i8> } @foo_ld2_v16i8(<16 x i1> %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: zip2 v1.16b, v0.16b, v0.16b ; CHECK-NEXT: zip1 v0.16b, v0.16b, v0.16b ; CHECK-NEXT: adrp x8, .LCPI0_0 -; CHECK-NEXT: ldr q2, [x8, :lo12:.LCPI0_0] -; CHECK-NEXT: shl v1.16b, v1.16b, #7 +; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI0_0] ; CHECK-NEXT: shl v0.16b, v0.16b, #7 -; CHECK-NEXT: cmlt v1.16b, v1.16b, #0 ; CHECK-NEXT: cmlt v0.16b, v0.16b, #0 -; CHECK-NEXT: and v1.16b, v1.16b, v2.16b -; CHECK-NEXT: and v0.16b, v0.16b, v2.16b -; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b +; CHECK-NEXT: and v0.16b, v0.16b, v1.16b ; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b -; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b ; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b -; CHECK-NEXT: addp v1.16b, v1.16b, v1.16b ; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b -; CHECK-NEXT: umov w9, v1.h[0] ; CHECK-NEXT: umov w8, v0.h[0] -; CHECK-NEXT: bfi w8, w9, #16, #16 +; CHECK-NEXT: bfi w8, w8, #16, #16 ; CHECK-NEXT: tbz w8, #0, .LBB0_2 ; CHECK-NEXT: // %bb.1: // %cond.load ; CHECK-NEXT: ldr b1, [x0] @@ -349,10 +341,11 @@ define { <8 x i16>, <8 x i16> } @foo_ld2_v8i16(<8 x i1> %mask, ptr %p) { define { <4 x float>, <4 x float> } @foo_ld2_v4f32(<4 x i1> %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_v4f32: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: zip1 v0.8h, v0.8h, v0.8h ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI2_0] -; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: shl v0.8b, v0.8b, #7 ; CHECK-NEXT: cmlt v0.8b, v0.8b, #0 ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b @@ -419,10 +412,11 @@ define { <4 x float>, <4 x float> } @foo_ld2_v4f32(<4 x i1> %mask, ptr %p) { define { <2 x double>, <2 x double> } @foo_ld2_v2f64(<2 x i1> %mask, ptr %p) { ; CHECK-LABEL: foo_ld2_v2f64: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI3_0 +; CHECK-NEXT: zip1 v0.4s, v0.4s, v0.4s ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI3_0] -; CHECK-NEXT: zip1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: shl v0.4h, v0.4h, #15 ; CHECK-NEXT: cmlt v0.4h, v0.4h, #0 ; CHECK-NEXT: and v0.8b, v0.8b, v1.8b @@ -453,8 +447,8 @@ define { <2 x double>, <2 x double> } @foo_ld2_v2f64(<2 x i1> %mask, ptr %p) { ; CHECK-NEXT: add x8, x0, #24 ; CHECK-NEXT: ld1 { v2.d }[1], [x8] ; CHECK-NEXT: .LBB3_8: // %else8 -; CHECK-NEXT: zip1 v0.2d, v1.2d, v2.2d -; CHECK-NEXT: zip2 v1.2d, v1.2d, v2.2d +; CHECK-NEXT: uzp1 v0.2d, v1.2d, v2.2d +; CHECK-NEXT: uzp2 v1.2d, v1.2d, v2.2d ; CHECK-NEXT: ret %interleaved.mask = call <4 x i1> @llvm.vector.interleave2.v4i1(<2 x i1> %mask, <2 x i1> %mask) %wide.masked.vec = call <4 x double> @llvm.masked.load.v4f64.p0(ptr %p, i32 8, <4 x i1> %interleaved.mask, <4 x double> poison) diff --git a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll index 972be0fca2089..5155be8cfed0d 100644 --- a/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll +++ b/llvm/test/CodeGen/AArch64/fixed_masked_interleaved_stores.ll @@ -5,26 +5,18 @@ define void @foo_st2_v16i8(<16 x i1> %mask, <16 x i8> %val1, <16 x i8> %val2, ptr %p) { ; CHECK-LABEL: foo_st2_v16i8: ; CHECK: // %bb.0: -; CHECK-NEXT: zip2 v3.16b, v0.16b, v0.16b ; CHECK-NEXT: zip1 v0.16b, v0.16b, v0.16b ; CHECK-NEXT: adrp x8, .LCPI0_0 -; CHECK-NEXT: ldr q4, [x8, :lo12:.LCPI0_0] -; CHECK-NEXT: shl v3.16b, v3.16b, #7 +; CHECK-NEXT: ldr q3, [x8, :lo12:.LCPI0_0] ; CHECK-NEXT: shl v0.16b, v0.16b, #7 -; CHECK-NEXT: cmlt v3.16b, v3.16b, #0 ; CHECK-NEXT: cmlt v0.16b, v0.16b, #0 -; CHECK-NEXT: and v3.16b, v3.16b, v4.16b -; CHECK-NEXT: and v0.16b, v0.16b, v4.16b -; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b +; CHECK-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b -; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b ; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b -; CHECK-NEXT: addp v3.16b, v3.16b, v3.16b ; CHECK-NEXT: addp v0.16b, v0.16b, v0.16b -; CHECK-NEXT: umov w9, v3.h[0] ; CHECK-NEXT: umov w8, v0.h[0] ; CHECK-NEXT: zip1 v0.16b, v1.16b, v2.16b -; CHECK-NEXT: bfi w8, w9, #16, #16 +; CHECK-NEXT: bfi w8, w8, #16, #16 ; CHECK-NEXT: tbnz w8, #0, .LBB0_33 ; CHECK-NEXT: // %bb.1: // %else ; CHECK-NEXT: tbnz w8, #1, .LBB0_34 @@ -345,10 +337,11 @@ define void @foo_st2_v8i16(<8 x i1> %mask, <8 x i16> %val1, <8 x i16> %val2, ptr define void @foo_st2_v4i32(<4 x i1> %mask, <4 x i32> %val1, <4 x i32> %val2, ptr %p) { ; CHECK-LABEL: foo_st2_v4i32: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI2_0 +; CHECK-NEXT: zip1 v0.8h, v0.8h, v0.8h ; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI2_0] -; CHECK-NEXT: zip1 v0.8b, v0.8b, v0.8b +; CHECK-NEXT: xtn v0.8b, v0.8h ; CHECK-NEXT: shl v0.8b, v0.8b, #7 ; CHECK-NEXT: cmlt v0.8b, v0.8b, #0 ; CHECK-NEXT: and v0.8b, v0.8b, v3.8b @@ -413,10 +406,11 @@ define void @foo_st2_v4i32(<4 x i1> %mask, <4 x i32> %val1, <4 x i32> %val2, ptr define void @foo_st2_v2i64(<2 x i1> %mask, <2 x i64> %val1, <2 x i64> %val2, ptr %p) { ; CHECK-LABEL: foo_st2_v2i64: ; CHECK: // %bb.0: -; CHECK-NEXT: uzp1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI3_0 +; CHECK-NEXT: zip1 v0.4s, v0.4s, v0.4s ; CHECK-NEXT: ldr d3, [x8, :lo12:.LCPI3_0] -; CHECK-NEXT: zip1 v0.4h, v0.4h, v0.4h +; CHECK-NEXT: xtn v0.4h, v0.4s ; CHECK-NEXT: shl v0.4h, v0.4h, #15 ; CHECK-NEXT: cmlt v0.4h, v0.4h, #0 ; CHECK-NEXT: and v0.8b, v0.8b, v3.8b diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll index ffc9067a78604..d81ad8206ba6c 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-deinterleave-load.ll @@ -12,24 +12,16 @@ define {<16 x i1>, <16 x i1>} @vector_deinterleave_load_v16i1_v32i1(ptr %p) { ; CHECK-NEXT: li a1, 32 ; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma ; CHECK-NEXT: vlm.v v0, (a0) -; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma ; CHECK-NEXT: vmv.v.i v8, 0 -; CHECK-NEXT: vmerge.vim v9, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, ma -; CHECK-NEXT: vslidedown.vi v0, v0, 2 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vnsrl.wi v10, v9, 0 -; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma ; CHECK-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vnsrl.wi v9, v9, 8 -; CHECK-NEXT: vnsrl.wi v11, v8, 0 -; CHECK-NEXT: vnsrl.wi v8, v8, 8 ; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma -; CHECK-NEXT: vslideup.vi v10, v11, 8 -; CHECK-NEXT: vslideup.vi v9, v8, 8 +; CHECK-NEXT: vnsrl.wi v10, v8, 0 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma ; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vmsne.vi v8, v9, 0 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vnsrl.wi v10, v8, 8 +; CHECK-NEXT: vsetvli zero, a1, e8, m2, ta, ma +; CHECK-NEXT: vmsne.vi v8, v10, 0 ; CHECK-NEXT: ret %vec = load <32 x i1>, ptr %p %deinterleaved.results = call {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1> %vec) diff --git a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll index bce071e988f0e..f002f4897bd66 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vector-deinterleave-fixed.ll @@ -8,24 +8,18 @@ define {<16 x i1>, <16 x i1>} @vector_deinterleave_v16i1_v32i1(<32 x i1> %vec) { ; CHECK-LABEL: vector_deinterleave_v16i1_v32i1: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: li a0, 32 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma ; CHECK-NEXT: vmv.v.i v8, 0 -; CHECK-NEXT: vmerge.vim v9, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 2, e8, mf4, ta, ma -; CHECK-NEXT: vslidedown.vi v0, v0, 2 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vnsrl.wi v10, v9, 0 -; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma ; CHECK-NEXT: vmerge.vim v8, v8, 1, v0 -; CHECK-NEXT: vsetivli zero, 8, e8, mf2, ta, ma -; CHECK-NEXT: vnsrl.wi v9, v9, 8 -; CHECK-NEXT: vnsrl.wi v11, v8, 0 -; CHECK-NEXT: vnsrl.wi v8, v8, 8 ; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma -; CHECK-NEXT: vslideup.vi v10, v11, 8 -; CHECK-NEXT: vslideup.vi v9, v8, 8 +; CHECK-NEXT: vnsrl.wi v10, v8, 0 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma ; CHECK-NEXT: vmsne.vi v0, v10, 0 -; CHECK-NEXT: vmsne.vi v8, v9, 0 +; CHECK-NEXT: vsetivli zero, 16, e8, m1, ta, ma +; CHECK-NEXT: vnsrl.wi v10, v8, 8 +; CHECK-NEXT: vsetvli zero, a0, e8, m2, ta, ma +; CHECK-NEXT: vmsne.vi v8, v10, 0 ; CHECK-NEXT: ret %retval = call {<16 x i1>, <16 x i1>} @llvm.vector.deinterleave2.v32i1(<32 x i1> %vec) ret {<16 x i1>, <16 x i1>} %retval @@ -74,22 +68,22 @@ ret {<4 x i32>, <4 x i32>} %retval define {<2 x i64>, <2 x i64>} @vector_deinterleave_v2i64_v4i64(<4 x i64> %vec) { ; V-LABEL: vector_deinterleave_v2i64_v4i64: ; V: # %bb.0: -; V-NEXT: vsetivli zero, 2, e64, m2, ta, ma -; V-NEXT: vslidedown.vi v10, v8, 2 -; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu -; V-NEXT: vmv.v.i v0, 1 +; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma +; V-NEXT: vmv.v.i v0, 2 +; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu +; V-NEXT: vslidedown.vi v10, v8, 1 +; V-NEXT: vslidedown.vi v10, v8, 2, v0.t +; V-NEXT: vslidedown.vi v8, v8, 1, v0.t ; V-NEXT: vmv1r.v v9, v10 -; V-NEXT: vslidedown.vi v9, v8, 1, v0.t -; V-NEXT: vslideup.vi v8, v10, 1 ; V-NEXT: ret ; ; ZVZIP-LABEL: vector_deinterleave_v2i64_v4i64: ; ZVZIP: # %bb.0: -; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; ZVZIP-NEXT: vunzipe.v v10, v8 -; ZVZIP-NEXT: vunzipo.v v11, v8 -; ZVZIP-NEXT: vmv.v.v v8, v10 -; ZVZIP-NEXT: vmv.v.v v9, v11 +; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; ZVZIP-NEXT: vunzipe.v v12, v8 +; ZVZIP-NEXT: vunzipo.v v14, v8 +; ZVZIP-NEXT: vmv1r.v v8, v12 +; ZVZIP-NEXT: vmv1r.v v9, v14 ; ZVZIP-NEXT: ret %retval = call {<2 x i64>, <2 x i64>} @llvm.vector.deinterleave2.v4i64(<4 x i64> %vec) ret {<2 x i64>, <2 x i64>} %retval @@ -98,44 +92,24 @@ ret {<2 x i64>, <2 x i64>} %retval define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) { ; V-LABEL: vector_deinterleave_v4i64_v8i64: ; V: # %bb.0: -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v0, 8 -; V-NEXT: vsetivli zero, 4, e64, m4, ta, ma -; V-NEXT: vslidedown.vi v16, v8, 4 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; V-NEXT: vslideup.vi v12, v16, 2 -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v10, 2 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu -; V-NEXT: vslideup.vi v12, v16, 1, v0.t -; V-NEXT: vmv2r.v v14, v8 -; V-NEXT: vmv1r.v v0, v10 -; V-NEXT: vslidedown.vi v14, v8, 1, v0.t -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v11, 12 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu -; V-NEXT: vslidedown.vi v18, v8, 1 -; V-NEXT: vmv1r.v v0, v11 -; V-NEXT: vmerge.vvm v12, v14, v12, v0 -; V-NEXT: vmv1r.v v0, v10 -; V-NEXT: vslidedown.vi v18, v8, 2, v0.t -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v0, 4 -; V-NEXT: vmv2r.v v8, v16 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu -; V-NEXT: vslideup.vi v8, v16, 1, v0.t -; V-NEXT: vmv1r.v v0, v11 -; V-NEXT: vmerge.vvm v10, v18, v8, v0 +; V-NEXT: li a0, 85 +; V-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; V-NEXT: vmv.s.x v16, a0 +; V-NEXT: vcompress.vm v12, v8, v16 +; V-NEXT: li a0, 170 +; V-NEXT: vmv.s.x v14, a0 +; V-NEXT: vcompress.vm v16, v8, v14 ; V-NEXT: vmv2r.v v8, v12 +; V-NEXT: vmv2r.v v10, v16 ; V-NEXT: ret ; ; ZVZIP-LABEL: vector_deinterleave_v4i64_v8i64: ; ZVZIP: # %bb.0: -; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; ZVZIP-NEXT: vunzipe.v v12, v8 -; ZVZIP-NEXT: vunzipo.v v14, v8 -; ZVZIP-NEXT: vmv.v.v v8, v12 -; ZVZIP-NEXT: vmv.v.v v10, v14 +; ZVZIP-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; ZVZIP-NEXT: vunzipe.v v16, v8 +; ZVZIP-NEXT: vunzipo.v v20, v8 +; ZVZIP-NEXT: vmv2r.v v8, v16 +; ZVZIP-NEXT: vmv2r.v v10, v20 ; ZVZIP-NEXT: ret %retval = call {<4 x i64>, <4 x i64>} @llvm.vector.deinterleave2.v8i64(<8 x i64> %vec) ret {<4 x i64>, <4 x i64>} %retval @@ -144,37 +118,18 @@ define {<4 x i64>, <4 x i64>} @vector_deinterleave_v4i64_v8i64(<8 x i64> %vec) { define {<8 x i64>, <8 x i64>} @vector_deinterleave_v8i64_v16i64(<16 x i64> %vec) { ; V-LABEL: vector_deinterleave_v8i64_v16i64: ; V: # %bb.0: -; V-NEXT: li a0, 85 -; V-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; V-NEXT: lui a0, 5 +; V-NEXT: addi a0, a0, 1365 +; V-NEXT: vsetivli zero, 16, e64, m8, ta, ma +; V-NEXT: vmv.s.x v24, a0 +; V-NEXT: vcompress.vm v16, v8, v24 +; V-NEXT: lui a0, 11 +; V-NEXT: addi a0, a0, -1366 ; V-NEXT: vmv.s.x v20, a0 -; V-NEXT: vcompress.vm v16, v8, v20 -; V-NEXT: vsetivli zero, 8, e64, m8, ta, ma -; V-NEXT: vslidedown.vi v24, v8, 8 -; V-NEXT: vsetivli zero, 8, e16, m1, ta, ma -; V-NEXT: vid.v v12 -; V-NEXT: vadd.vv v20, v12, v12 -; V-NEXT: vmv.v.i v0, -16 -; V-NEXT: vadd.vi v12, v20, -8 -; V-NEXT: vsetvli zero, zero, e64, m4, ta, mu -; V-NEXT: vrgatherei16.vv v16, v24, v12, v0.t -; V-NEXT: li a0, 170 -; V-NEXT: vmv.s.x v21, a0 -; V-NEXT: vcompress.vm v12, v8, v21 -; V-NEXT: vsetvli zero, zero, e16, m1, ta, ma -; V-NEXT: vadd.vi v8, v20, -7 -; V-NEXT: vsetvli zero, zero, e64, m4, ta, mu -; V-NEXT: vrgatherei16.vv v12, v24, v8, v0.t -; V-NEXT: vmv.v.v v8, v16 +; V-NEXT: vcompress.vm v24, v8, v20 +; V-NEXT: vmv4r.v v8, v16 +; V-NEXT: vmv4r.v v12, v24 ; V-NEXT: ret -; -; ZVZIP-LABEL: vector_deinterleave_v8i64_v16i64: -; ZVZIP: # %bb.0: -; ZVZIP-NEXT: vsetivli zero, 8, e64, m4, ta, ma -; ZVZIP-NEXT: vunzipe.v v16, v8 -; ZVZIP-NEXT: vunzipo.v v20, v8 -; ZVZIP-NEXT: vmv.v.v v8, v16 -; ZVZIP-NEXT: vmv.v.v v12, v20 -; ZVZIP-NEXT: ret %retval = call {<8 x i64>, <8 x i64>} @llvm.vector.deinterleave2.v16i64(<16 x i64> %vec) ret {<8 x i64>, <8 x i64>} %retval } @@ -800,22 +755,22 @@ ret {<4 x float>, <4 x float>} %retval define {<2 x double>, <2 x double>} @vector_deinterleave_v2f64_v4f64(<4 x double> %vec) { ; V-LABEL: vector_deinterleave_v2f64_v4f64: ; V: # %bb.0: -; V-NEXT: vsetivli zero, 2, e64, m2, ta, ma -; V-NEXT: vslidedown.vi v10, v8, 2 -; V-NEXT: vsetivli zero, 2, e64, m1, ta, mu -; V-NEXT: vmv.v.i v0, 1 +; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma +; V-NEXT: vmv.v.i v0, 2 +; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu +; V-NEXT: vslidedown.vi v10, v8, 1 +; V-NEXT: vslidedown.vi v10, v8, 2, v0.t +; V-NEXT: vslidedown.vi v8, v8, 1, v0.t ; V-NEXT: vmv1r.v v9, v10 -; V-NEXT: vslidedown.vi v9, v8, 1, v0.t -; V-NEXT: vslideup.vi v8, v10, 1 ; V-NEXT: ret ; ; ZVZIP-LABEL: vector_deinterleave_v2f64_v4f64: ; ZVZIP: # %bb.0: -; ZVZIP-NEXT: vsetivli zero, 2, e64, m1, ta, ma -; ZVZIP-NEXT: vunzipe.v v10, v8 -; ZVZIP-NEXT: vunzipo.v v11, v8 -; ZVZIP-NEXT: vmv.v.v v8, v10 -; ZVZIP-NEXT: vmv.v.v v9, v11 +; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma +; ZVZIP-NEXT: vunzipe.v v12, v8 +; ZVZIP-NEXT: vunzipo.v v14, v8 +; ZVZIP-NEXT: vmv1r.v v8, v12 +; ZVZIP-NEXT: vmv1r.v v9, v14 ; ZVZIP-NEXT: ret %retval = call {<2 x double>, <2 x double>} @llvm.vector.deinterleave2.v4f64(<4 x double> %vec) ret {<2 x double>, <2 x double>} %retval @@ -824,44 +779,24 @@ ret {<2 x double>, <2 x double>} %retval define {<4 x double>, <4 x double>} @vector_deinterleave_v4f64_v8f64(<8 x double> %vec) { ; V-LABEL: vector_deinterleave_v4f64_v8f64: ; V: # %bb.0: -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v0, 8 -; V-NEXT: vsetivli zero, 4, e64, m4, ta, ma -; V-NEXT: vslidedown.vi v16, v8, 4 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; V-NEXT: vslideup.vi v12, v16, 2 -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v10, 2 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu -; V-NEXT: vslideup.vi v12, v16, 1, v0.t -; V-NEXT: vmv2r.v v14, v8 -; V-NEXT: vmv1r.v v0, v10 -; V-NEXT: vslidedown.vi v14, v8, 1, v0.t -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v11, 12 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu -; V-NEXT: vslidedown.vi v18, v8, 1 -; V-NEXT: vmv1r.v v0, v11 -; V-NEXT: vmerge.vvm v12, v14, v12, v0 -; V-NEXT: vmv1r.v v0, v10 -; V-NEXT: vslidedown.vi v18, v8, 2, v0.t -; V-NEXT: vsetivli zero, 1, e8, mf8, ta, ma -; V-NEXT: vmv.v.i v0, 4 -; V-NEXT: vmv2r.v v8, v16 -; V-NEXT: vsetivli zero, 4, e64, m2, ta, mu -; V-NEXT: vslideup.vi v8, v16, 1, v0.t -; V-NEXT: vmv1r.v v0, v11 -; V-NEXT: vmerge.vvm v10, v18, v8, v0 +; V-NEXT: li a0, 85 +; V-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; V-NEXT: vmv.s.x v16, a0 +; V-NEXT: vcompress.vm v12, v8, v16 +; V-NEXT: li a0, 170 +; V-NEXT: vmv.s.x v14, a0 +; V-NEXT: vcompress.vm v16, v8, v14 ; V-NEXT: vmv2r.v v8, v12 +; V-NEXT: vmv2r.v v10, v16 ; V-NEXT: ret ; ; ZVZIP-LABEL: vector_deinterleave_v4f64_v8f64: ; ZVZIP: # %bb.0: -; ZVZIP-NEXT: vsetivli zero, 4, e64, m2, ta, ma -; ZVZIP-NEXT: vunzipe.v v12, v8 -; ZVZIP-NEXT: vunzipo.v v14, v8 -; ZVZIP-NEXT: vmv.v.v v8, v12 -; ZVZIP-NEXT: vmv.v.v v10, v14 +; ZVZIP-NEXT: vsetivli zero, 8, e64, m4, ta, ma +; ZVZIP-NEXT: vunzipe.v v16, v8 +; ZVZIP-NEXT: vunzipo.v v20, v8 +; ZVZIP-NEXT: vmv2r.v v8, v16 +; ZVZIP-NEXT: vmv2r.v v10, v20 ; ZVZIP-NEXT: ret %retval = call {<4 x double>, <4 x double>} @llvm.vector.deinterleave2.v8f64(<8 x double> %vec) ret {<4 x double>, <4 x double>} %retval