diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 51be0e66b19b0..717e43d9118f2 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -27784,8 +27784,6 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, auto *MemN = cast(N); if (IsScalable) { - if (NumParts == 3) - return SDValue(); SDValue Pred; if (IsMasked) { Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts); @@ -27796,8 +27794,18 @@ performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, Pred = DAG.getConstant(1, DL, PredVT); } - const Intrinsic::ID IID = - NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4; + Intrinsic::ID IID; + switch (NumParts) { + case 2: + IID = Intrinsic::aarch64_sve_st2; + break; + case 3: + IID = Intrinsic::aarch64_sve_st3; + break; + case 4: + IID = Intrinsic::aarch64_sve_st4; + break; + } SmallVector Ops; Ops.append({Chain, DAG.getConstant(IID, DL, MVT::i32)}); Ops.append(ValueInterleaveOps); @@ -30848,8 +30856,6 @@ static SDValue performVectorDeinterleaveCombine( SDValue Res; MemSDNode *MemNode = dyn_cast(WideVec); if (IsScalable) { - if (NumParts == 3) - return SDValue(); SDValue Chain, BasePtr, Pred; if (auto *MaskedLoad = dyn_cast(WideVec)) { // Bail out if the masked load has an unexpected number of uses, since we @@ -30883,8 +30889,18 @@ static SDValue performVectorDeinterleaveCombine( BasePtr = Load->getBasePtr(); } - const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_ld2_sret - : Intrinsic::aarch64_sve_ld4_sret; + Intrinsic::ID IID; + switch (NumParts) { + case 2: + IID = Intrinsic::aarch64_sve_ld2_sret; + break; + case 3: + IID = Intrinsic::aarch64_sve_ld3_sret; + break; + case 4: + IID = Intrinsic::aarch64_sve_ld4_sret; + break; + } SDValue NewLdOps[] = {Chain, DAG.getConstant(IID, DL, MVT::i32), Pred, BasePtr}; Res = DAG.getMemIntrinsicNode(ISD::INTRINSIC_W_CHAIN, DL, ResVTList, @@ -34352,12 +34368,40 @@ AArch64TargetLowering::LowerFixedLengthIntToFPToSVE(SDValue Op, } } +static SDValue lowerPredicateInterleaveWithPackedVectors(SDValue Op, + SelectionDAG &DAG) { + SDLoc DL(Op); + EVT OpVT = Op.getValueType(); + assert(OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 && + Op->getNumOperands() == 3 && "Unexpected predicate interleave"); + + EVT PackedVT = getPackedSVEVectorVT(OpVT.getVectorElementCount()); + SmallVector PackedOps; + for (SDValue V : Op->ops()) + PackedOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, PackedVT, V)); + + SmallVector PackedVTs(3, PackedVT); + SDValue PackedOp = + DAG.getNode(Op.getOpcode(), DL, DAG.getVTList(PackedVTs), PackedOps); + + SDValue Zero = DAG.getConstant(0, DL, PackedVT); + SmallVector Results; + for (unsigned I = 0; I != 3; ++I) + Results.push_back( + DAG.getSetCC(DL, OpVT, PackedOp.getValue(I), Zero, ISD::SETNE)); + return DAG.getMergeValues(Results, DL); +} + SDValue AArch64TargetLowering::LowerVECTOR_DEINTERLEAVE(SDValue Op, SelectionDAG &DAG) const { SDLoc DL(Op); EVT OpVT = Op.getValueType(); + if (OpVT.isScalableVector() && Op->getNumOperands() == 3) + if (OpVT.getVectorElementType() == MVT::i1) + return lowerPredicateInterleaveWithPackedVectors(Op, DAG); + if (OpVT.isFixedLengthVector() && Op->getNumOperands() == 3) { Align Alignment = DAG.getReducedAlign(OpVT, /*UseABI=*/false); SDValue StackPtr = @@ -34468,6 +34512,10 @@ SDValue AArch64TargetLowering::LowerVECTOR_INTERLEAVE(SDValue Op, SDLoc DL(Op); EVT OpVT = Op.getValueType(); + if (OpVT.isScalableVector() && Op->getNumOperands() == 3) + if (OpVT.getVectorElementType() == MVT::i1) + return lowerPredicateInterleaveWithPackedVectors(Op, DAG); + if (OpVT.isFixedLengthVector() && Op->getNumOperands() == 3) { Align Alignment = DAG.getReducedAlign(OpVT, /*UseABI=*/false); SDValue StackPtr = diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll index ee2d482d9ffb5..f5129c7c57826 100644 --- a/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll +++ b/llvm/test/CodeGen/AArch64/scalable_masked_deinterleaved_loads.ll @@ -45,6 +45,50 @@ define { , } @foo_ld2_nxv2f64(, } %deinterleaved.vec } +define { , , } @foo_ld3_nxv16i8( %mask, ptr %p) { +; CHECK-LABEL: foo_ld3_nxv16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: ld3b { z0.b - z2.b }, p0/z, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv48i1( %mask, %mask, %mask) + %wide.masked.vec = call @llvm.masked.load.nxv48i8(ptr %p, i32 1, %interleaved.mask, poison) + %deinterleaved.vec = call { , , } @llvm.vector.deinterleave3.nxv48i8( %wide.masked.vec) + ret { , , } %deinterleaved.vec +} + +define { , , } @foo_ld3_nxv8i16( %mask, ptr %p) { +; CHECK-LABEL: foo_ld3_nxv8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: ld3h { z0.h - z2.h }, p0/z, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv24i1( %mask, %mask, %mask) + %wide.masked.vec = call @llvm.masked.load.nxv24i16(ptr %p, i32 1, %interleaved.mask, poison) + %deinterleaved.vec = call { , , } @llvm.vector.deinterleave3.nxv24i16( %wide.masked.vec) + ret { , , } %deinterleaved.vec +} + +define { , , } @foo_ld3_nxv4i32( %mask, ptr %p) { +; CHECK-LABEL: foo_ld3_nxv4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: ld3w { z0.s - z2.s }, p0/z, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv12i1( %mask, %mask, %mask) + %wide.masked.vec = call @llvm.masked.load.nxv12i32(ptr %p, i32 1, %interleaved.mask, poison) + %deinterleaved.vec = call { , , } @llvm.vector.deinterleave3.nxv12i32( %wide.masked.vec) + ret { , , } %deinterleaved.vec +} + +define { , , } @foo_ld3_nxv2i64( %mask, ptr %p) { +; CHECK-LABEL: foo_ld3_nxv2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: ld3d { z0.d - z2.d }, p0/z, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv6i1( %mask, %mask, %mask) + %wide.masked.vec = call @llvm.masked.load.nxv6i64(ptr %p, i32 1, %interleaved.mask, poison) + %deinterleaved.vec = call { , , } @llvm.vector.deinterleave3.nxv6i64( %wide.masked.vec) + ret { , , } %deinterleaved.vec +} + define { , , , } @foo_ld4_nxv16i8( %mask, ptr %p) { ; CHECK-LABEL: foo_ld4_nxv16i8: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll index fc2f64b756959..766777925e909 100644 --- a/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll +++ b/llvm/test/CodeGen/AArch64/scalable_masked_interleaved_stores.ll @@ -54,6 +54,62 @@ define void @foo_st2_nxv2i64( %mask, %val1, ret void } +define void @foo_st3_nxv16i8( %mask, %val1, %val2, %val3, ptr %p) { +; CHECK-LABEL: foo_st3_nxv16i8: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: st3b { z0.b - z2.b }, p0, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv48i1( %mask, %mask, %mask) + %interleaved.value = call @llvm.vector.interleave3.nxv48i8( %val1, %val2, %val3) + call void @llvm.masked.store.nxv48i8.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) + ret void +} + +define void @foo_st3_nxv8i16( %mask, %val1, %val2, %val3, ptr %p) { +; CHECK-LABEL: foo_st3_nxv8i16: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv24i1( %mask, %mask, %mask) + %interleaved.value = call @llvm.vector.interleave3.nxv24i16( %val1, %val2, %val3) + call void @llvm.masked.store.nxv24i16.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) + ret void +} + +define void @foo_st3_nxv4i32( %mask, %val1, %val2, %val3, ptr %p) { +; CHECK-LABEL: foo_st3_nxv4i32: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv12i1( %mask, %mask, %mask) + %interleaved.value = call @llvm.vector.interleave3.nxv12i32( %val1, %val2, %val3) + call void @llvm.masked.store.nxv12i32.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) + ret void +} + +define void @foo_st3_nxv2i64( %mask, %val1, %val2, %val3, ptr %p) { +; CHECK-LABEL: foo_st3_nxv2i64: +; CHECK: // %bb.0: +; CHECK-NEXT: // kill: def $z2 killed $z2 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1_z2 def $z0_z1_z2 +; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [x0] +; CHECK-NEXT: ret + %interleaved.mask = call @llvm.vector.interleave3.nxv6i1( %mask, %mask, %mask) + %interleaved.value = call @llvm.vector.interleave3.nxv6i64( %val1, %val2, %val3) + call void @llvm.masked.store.nxv6i64.p0( %interleaved.value, ptr %p, i32 1, %interleaved.mask) + ret void +} + define void @foo_st4_nxv16i8( %mask, %val1, %val2, %val3, %val4, ptr %p) { ; CHECK-LABEL: foo_st4_nxv16i8: ; CHECK: // %bb.0: diff --git a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll index f0d4e86752dec..62d62de9e3971 100644 --- a/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-deinterleave.ll @@ -712,6 +712,58 @@ define {, } @vector_deinterleave_nxv2i1_nxv4i1 ret {, } %retval } +define {, , } @deinterleave3_nxv16i1_nxv24i1( %vec) { +; CHECK-LABEL: deinterleave3_nxv16i1_nxv24i1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: punpklo p1.h, p1.b +; CHECK-NEXT: punpkhi p3.h, p0.b +; CHECK-NEXT: punpklo p0.h, p0.b +; CHECK-NEXT: mov z0.h, p1/z, #1 // =0x1 +; CHECK-NEXT: mov z1.h, p3/z, #1 // =0x1 +; CHECK-NEXT: ptrue p2.h +; CHECK-NEXT: mov z2.h, p0/z, #1 // =0x1 +; CHECK-NEXT: str z0, [sp, #2, mul vl] +; CHECK-NEXT: str z1, [sp, #1, mul vl] +; CHECK-NEXT: str z2, [sp] +; CHECK-NEXT: ld3h { z0.h - z2.h }, p2/z, [sp] +; CHECK-NEXT: cmpne p0.h, p2/z, z0.h, #0 +; CHECK-NEXT: cmpne p1.h, p2/z, z1.h, #0 +; CHECK-NEXT: cmpne p2.h, p2/z, z2.h, #0 +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %retval = call {, , } @llvm.vector.deinterleave3.nxv24i1( %vec) + ret {, , } %retval +} + +define {, , } @deinterleave3_nxv16i1_nxv48i1( %vec) { +; CHECK-LABEL: deinterleave3_nxv16i1_nxv48i1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: mov z0.b, p2/z, #1 // =0x1 +; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1 +; CHECK-NEXT: mov z2.b, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p2.b +; CHECK-NEXT: str z0, [sp, #2, mul vl] +; CHECK-NEXT: str z1, [sp, #1, mul vl] +; CHECK-NEXT: str z2, [sp] +; CHECK-NEXT: ld3b { z0.b - z2.b }, p2/z, [sp] +; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0 +; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; CHECK-NEXT: cmpne p2.b, p2/z, z2.b, #0 +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %retval = call {, , } @llvm.vector.deinterleave3.nxv48i1( %vec) + ret {, , } %retval +} ; Split illegal types diff --git a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll index c497ba6f7008b..32689d88c09ea 100644 --- a/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll +++ b/llvm/test/CodeGen/AArch64/sve-vector-interleave.ll @@ -659,6 +659,114 @@ define @interleave2_nxv2i1( %vec0, %retval } +define @interleave3_nxv6i1( %vec0, %vec1, %vec2) { +; CHECK-LABEL: interleave3_nxv6i1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: mov z2.d, p2/z, #1 // =0x1 +; CHECK-NEXT: mov z1.d, p1/z, #1 // =0x1 +; CHECK-NEXT: mov z0.d, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p0.d +; CHECK-NEXT: st3d { z0.d - z2.d }, p0, [sp] +; CHECK-NEXT: ldr z0, [sp, #1, mul vl] +; CHECK-NEXT: ldr z1, [sp] +; CHECK-NEXT: ldr z2, [sp, #2, mul vl] +; CHECK-NEXT: cmpne p1.d, p0/z, z0.d, #0 +; CHECK-NEXT: cmpne p2.d, p0/z, z1.d, #0 +; CHECK-NEXT: cmpne p3.d, p0/z, z2.d, #0 +; CHECK-NEXT: uzp1 p0.s, p2.s, p1.s +; CHECK-NEXT: uzp1 p1.s, p3.s, p0.s +; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave3.nxv6i1( %vec0, %vec1, %vec2) + ret %retval +} + +define @interleave3_nxv12i1( %vec0, %vec1, %vec2) { +; CHECK-LABEL: interleave3_nxv12i1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: mov z2.s, p2/z, #1 // =0x1 +; CHECK-NEXT: mov z1.s, p1/z, #1 // =0x1 +; CHECK-NEXT: mov z0.s, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p0.s +; CHECK-NEXT: st3w { z0.s - z2.s }, p0, [sp] +; CHECK-NEXT: ldr z0, [sp, #1, mul vl] +; CHECK-NEXT: ldr z1, [sp] +; CHECK-NEXT: ldr z2, [sp, #2, mul vl] +; CHECK-NEXT: cmpne p1.s, p0/z, z0.s, #0 +; CHECK-NEXT: cmpne p2.s, p0/z, z1.s, #0 +; CHECK-NEXT: cmpne p3.s, p0/z, z2.s, #0 +; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h +; CHECK-NEXT: uzp1 p1.h, p3.h, p0.h +; CHECK-NEXT: uzp1 p0.b, p0.b, p1.b +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave3.nxv12i1( %vec0, %vec1, %vec2) + ret %retval +} + +define @interleave3_nxv24i1( %vec0, %vec1, %vec2) { +; CHECK-LABEL: interleave3_nxv24i1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: mov z2.h, p2/z, #1 // =0x1 +; CHECK-NEXT: mov z1.h, p1/z, #1 // =0x1 +; CHECK-NEXT: mov z0.h, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p0.h +; CHECK-NEXT: st3h { z0.h - z2.h }, p0, [sp] +; CHECK-NEXT: ldr z0, [sp, #1, mul vl] +; CHECK-NEXT: ldr z1, [sp] +; CHECK-NEXT: ldr z2, [sp, #2, mul vl] +; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0 +; CHECK-NEXT: cmpne p2.h, p0/z, z1.h, #0 +; CHECK-NEXT: cmpne p3.h, p0/z, z2.h, #0 +; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b +; CHECK-NEXT: uzp1 p1.b, p3.b, p0.b +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave3.nxv24i1( %vec0, %vec1, %vec2) + ret %retval +} + +define @interleave3_nxv48i1( %vec0, %vec1, %vec2) { +; CHECK-LABEL: interleave3_nxv48i1: +; CHECK: // %bb.0: +; CHECK-NEXT: str x29, [sp, #-16]! // 8-byte Folded Spill +; CHECK-NEXT: addvl sp, sp, #-3 +; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x48, 0x1e, 0x22 // sp + 16 + 24 * VG +; CHECK-NEXT: .cfi_offset w29, -16 +; CHECK-NEXT: mov z2.b, p2/z, #1 // =0x1 +; CHECK-NEXT: mov z1.b, p1/z, #1 // =0x1 +; CHECK-NEXT: mov z0.b, p0/z, #1 // =0x1 +; CHECK-NEXT: ptrue p2.b +; CHECK-NEXT: st3b { z0.b - z2.b }, p2, [sp] +; CHECK-NEXT: ldr z0, [sp] +; CHECK-NEXT: ldr z1, [sp, #1, mul vl] +; CHECK-NEXT: ldr z2, [sp, #2, mul vl] +; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0 +; CHECK-NEXT: cmpne p1.b, p2/z, z1.b, #0 +; CHECK-NEXT: cmpne p2.b, p2/z, z2.b, #0 +; CHECK-NEXT: addvl sp, sp, #3 +; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload +; CHECK-NEXT: ret + %retval = call @llvm.vector.interleave3.nxv48i1( %vec0, %vec1, %vec2) + ret %retval +} + ; Split illegal type size define @interleave2_nxv16i32( %vec0, %vec1) {