[AArch64][SVE] Enable factor-3 scalable interleave operations - #200424
[AArch64][SVE] Enable factor-3 scalable interleave operations#200424Harry-Ramsey wants to merge 1 commit into
Conversation
|
@llvm/pr-subscribers-llvm-transforms @llvm/pr-subscribers-backend-aarch64 Author: Harry Ramsey (Harry-Ramsey) ChangesThis commit enables LoopVectorize to lower scalable vector.[de]interleave3 to use SVE ld3/st3 instructions. Patch is 20.25 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/200424.diff 4 Files Affected:
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index 730bec428a38e..8708cb5038c91 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -5129,11 +5129,8 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost(
if (VecTy->isScalableTy() && !ST->hasSVE())
return InstructionCost::getInvalid();
- // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we
- // only have lowering for power-of-2 factors.
- // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in
- // InterleavedAccessPass for ld3/st3
- if (VecTy->isScalableTy() && !isPowerOf2_32(Factor))
+ // Scalable VFs emit vector.[de]interleave intrinsics.
+ if (VecTy->isScalableTy() && Factor > TLI->getMaxSupportedInterleaveFactor())
return InstructionCost::getInvalid();
// Vectorization for masked interleaved accesses is only enabled for scalable
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
index 31a1d95dedd3c..7c9762d57d443 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/force-target-instruction-cost.ll
@@ -439,27 +439,40 @@ define void @interleave_group(ptr %dst) #1 {
; COST1-NEXT: [[ITER_CHECK:.*:]]
; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST1: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST1-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64()
+; COST1-NEXT: [[TMP25:%.*]] = shl nuw i64 [[TMP24]], 5
+; COST1-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP25]]
+; COST1-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST1: [[VECTOR_PH]]:
+; COST1-NEXT: [[TMP26:%.*]] = shl nuw i64 [[TMP24]], 4
+; COST1-NEXT: [[TMP27:%.*]] = shl nuw i64 [[TMP26]], 1
+; COST1-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP27]]
+; COST1-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST1-NEXT: br label %[[VECTOR_BODY:.*]]
; COST1: [[VECTOR_BODY]]:
; COST1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
-; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], 16
+; COST1-NEXT: [[TMP28:%.*]] = add i64 [[TMP26]], 0
+; COST1-NEXT: [[TMP5:%.*]] = mul i64 [[TMP28]], 1
+; COST1-NEXT: [[TMP0:%.*]] = add i64 [[INDEX]], [[TMP5]]
; COST1-NEXT: [[TMP1:%.*]] = mul i64 [[INDEX]], 3
; COST1-NEXT: [[TMP2:%.*]] = mul i64 [[TMP0]], 3
; COST1-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP1]]
; COST1-NEXT: [[TMP4:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP2]]
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1
-; COST1-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP4]], align 1
-; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32
-; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP3]], align 1
+; COST1-NEXT: [[INTERLEAVED_VEC1:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST1-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC1]], ptr [[TMP4]], align 1
+; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP27]]
+; COST1-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST1-NEXT: br i1 [[TMP29]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST1: [[MIDDLE_BLOCK]]:
-; COST1-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST1-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST1-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST1: [[VEC_EPILOG_ITER_CHECK]]:
-; COST1-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST1-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST1-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST1: [[VEC_EPILOG_PH]]:
-; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST1-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST1-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST1-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST1-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
@@ -509,23 +522,32 @@ define void @interleave_group(ptr %dst) #1 {
; COST10-NEXT: [[ITER_CHECK:.*:]]
; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH:.*]], label %[[VECTOR_MAIN_LOOP_ITER_CHECK:.*]]
; COST10: [[VECTOR_MAIN_LOOP_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
+; COST10-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64()
+; COST10-NEXT: [[TMP22:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 101, [[TMP22]]
+; COST10-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[VEC_EPILOG_PH:.*]], label %[[VECTOR_PH:.*]]
; COST10: [[VECTOR_PH]]:
+; COST10-NEXT: [[TMP2:%.*]] = shl nuw i64 [[TMP21]], 4
+; COST10-NEXT: [[N_MOD_VF:%.*]] = urem i64 101, [[TMP2]]
+; COST10-NEXT: [[N_VEC:%.*]] = sub i64 101, [[N_MOD_VF]]
; COST10-NEXT: br label %[[VECTOR_BODY:.*]]
; COST10: [[VECTOR_BODY]]:
; COST10-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
; COST10-NEXT: [[TMP0:%.*]] = mul i64 [[INDEX]], 3
; COST10-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr [[DST]], i64 [[TMP0]]
-; COST10-NEXT: store <48 x i8> zeroinitializer, ptr [[TMP1]], align 1
-; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 16
-; COST10-NEXT: [[TMP2:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96
-; COST10-NEXT: br i1 [[TMP2]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
+; COST10-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer)
+; COST10-NEXT: store <vscale x 48 x i8> [[INTERLEAVED_VEC]], ptr [[TMP1]], align 1
+; COST10-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP2]]
+; COST10-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; COST10-NEXT: br i1 [[TMP23]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]]
; COST10: [[MIDDLE_BLOCK]]:
-; COST10-NEXT: br i1 false, [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
+; COST10-NEXT: [[CMP_N:%.*]] = icmp eq i64 101, [[N_VEC]]
+; COST10-NEXT: br i1 [[CMP_N]], [[EXIT:label %.*]], label %[[VEC_EPILOG_ITER_CHECK:.*]]
; COST10: [[VEC_EPILOG_ITER_CHECK]]:
-; COST10-NEXT: br i1 false, label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
+; COST10-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_MOD_VF]], 4
+; COST10-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label %[[VEC_EPILOG_SCALAR_PH]], label %[[VEC_EPILOG_PH]], !prof [[PROF3]]
; COST10: [[VEC_EPILOG_PH]]:
-; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ 96, %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
+; COST10-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], %[[VEC_EPILOG_ITER_CHECK]] ], [ 0, %[[VECTOR_MAIN_LOOP_ITER_CHECK]] ]
; COST10-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[BC_RESUME_VAL]], i64 0
; COST10-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
; COST10-NEXT: [[INDUCTION:%.*]] = add <4 x i64> [[BROADCAST_SPLAT]], <i64 0, i64 1, i64 2, i64 3>
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
index 0923260c82a86..31d5a5aeabf3b 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll
@@ -1332,8 +1332,6 @@ end:
; dst[i].z = a[i].z << b[i].z;
; }
;
-; TODO: Support scalable interleave groups once we can also codegen
-; @llvm.[de]interleave3
%struct.xyz = type { i32, i32, i32 }
define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr readonly %a, ptr readonly %b) {
@@ -1347,36 +1345,28 @@ define void @interleave_deinterleave_factor3(ptr writeonly noalias %dst, ptr rea
; CHECK-NEXT: [[TMP3:%.*]] = shl nuw i64 [[TMP0]], 2
; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]]
; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]]
-; CHECK-NEXT: [[TMP6:%.*]] = call <vscale x 4 x i64> @llvm.stepvector.nxv4i64()
-; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement <vscale x 4 x i64> poison, i64 [[TMP3]], i64 0
-; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector <vscale x 4 x i64> [[DOTSPLATINSERT]], <vscale x 4 x i64> poison, <vscale x 4 x i32> zeroinitializer
; CHECK-NEXT: br label [[VECTOR_BODY:%.*]]
; CHECK: vector.body:
; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[VEC_IND:%.*]] = phi <vscale x 4 x i64> [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ]
-; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP7:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP19]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: [[TMP11:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP20]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds [[STRUCT_XYZ:%.*]], ptr [[A:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC:%.*]] = load <vscale x 12 x i32>, ptr [[TMP4]], align 4
+; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC]])
+; CHECK-NEXT: [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 0
+; CHECK-NEXT: [[TMP8:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 1
+; CHECK-NEXT: [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC]], 2
+; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[B:%.*]], i64 [[INDEX]]
+; CHECK-NEXT: [[WIDE_VEC1:%.*]] = load <vscale x 12 x i32>, ptr [[TMP10]], align 4
+; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.vector.deinterleave3.nxv12i32(<vscale x 12 x i32> [[WIDE_VEC1]])
+; CHECK-NEXT: [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 0
+; CHECK-NEXT: [[TMP12:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 1
+; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[STRIDED_VEC2]], 2
; CHECK-NEXT: [[TMP14:%.*]] = add nsw <vscale x 4 x i32> [[TMP11]], [[TMP7]]
-; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], <vscale x 4 x i64> [[VEC_IND]]
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x ptr> align 4 [[TMP10]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP19]], i64 4
-; CHECK-NEXT: [[TMP8:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP21]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP20]], i64 4
-; CHECK-NEXT: [[TMP12:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP22]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
+; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds [[STRUCT_XYZ]], ptr [[DST:%.*]], i64 [[INDEX]]
; CHECK-NEXT: [[TMP16:%.*]] = sub nsw <vscale x 4 x i32> [[TMP8]], [[TMP12]]
-; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP10]], i64 4
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP16]], <vscale x 4 x ptr> align 4 [[TMP23]], <vscale x 4 x i1> splat (i1 true))
-; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP19]], i64 8
-; CHECK-NEXT: [[TMP9:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP15]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
-; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP20]], i64 8
-; CHECK-NEXT: [[TMP13:%.*]] = call <vscale x 4 x i32> @llvm.masked.gather.nxv4i32.nxv4p0(<vscale x 4 x ptr> align 4 [[TMP24]], <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> poison)
; CHECK-NEXT: [[TMP17:%.*]] = shl <vscale x 4 x i32> [[TMP9]], [[TMP13]]
-; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds nuw i8, <vscale x 4 x ptr> [[TMP10]], i64 8
-; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0(<vscale x 4 x i32> [[TMP17]], <vscale x 4 x ptr> align 4 [[TMP25]], <vscale x 4 x i1> splat (i1 true))
+; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 12 x i32> @llvm.vector.interleave3.nxv12i32(<vscale x 4 x i32> [[TMP14]], <vscale x 4 x i32> [[TMP16]], <vscale x 4 x i32> [[TMP17]])
+; CHECK-NEXT: store <vscale x 12 x i32> [[INTERLEAVED_VEC]], ptr [[TMP15]], align 4
; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP3]]
-; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <vscale x 4 x i64> [[VEC_IND]], [[DOTSPLAT]]
; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP41:![0-9]+]]
; CHECK: middle.block:
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
index 21d5a2e74028b..43a38193d5585 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-option.ll
@@ -283,39 +283,45 @@ for.end:
define void @interleave(ptr noalias %dst, ptr noalias %src, i64 %n) #0 {
; CHECK-NOTF-LABEL: @interleave(
; CHECK-NOTF: vector.body:
-; CHECK-NOTF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NOTF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NOTF: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-NOTF: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-NOTF: %{{.*}} = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32
+; CHECK-NOTF: store <vscale x 12 x float>
; CHECK-TF-LABEL: @interleave(
; CHECK-TF: vector.body:
-; CHECK-TF: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF: %{{.*}} = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32
+; CHECK-TF: store <vscale x 12 x float>
; CHECK-TF-NORED-LABEL: @interleave(
; CHECK-TF-NORED: vector.body:
-; CHECK-TF-NORED: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NORED: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NORED: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF-NORED: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF-NORED: %{{.*}} = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32
+; CHECK-TF-NORED: store <vscale x 12 x float>
; CHECK-TF-NOREC-LABEL: @interleave(
; CHECK-TF-NOREC: vector.body:
-; CHECK-TF-NOREC: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREC: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREC: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF-NOREC: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF-NOREC: %{{.*}} = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32
+; CHECK-TF-NOREC: store <vscale x 12 x float>
; CHECK-TF-NOREV-LABEL: @interleave(
; CHECK-TF-NOREV: vector.body:
-; CHECK-TF-NOREV: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-TF-NOREV: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-TF-NOREV: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-TF-NOREV: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llvm.vector.deinterleave2.nxv8f32(<vscale x 8 x float> %[[LOAD]])
+; CHECK-TF-NOREV: %{{.*}} = call <vscale x 12 x float> @llvm.vector.interleave3.nxv12f32
+; CHECK-TF-NOREV: store <vscale x 12 x float>
; CHECK-NEOVERSE-V1-LABEL: @interleave(
; CHECK-NEOVERSE-V1: vector.body:
-; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
-; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
+; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <vscale x 8 x float>, ptr
+; CHECK-NEOVERSE-V1: %{{.*}} = call { <vscale x 4 x float>, <vscale x 4 x float> } @llv...
[truncated]
|
|
I assume this patch is waiting on the codegen side of things? |
66f28b8 to
ff301a9
Compare
|
✅ With the latest revision this PR passed the C/C++ code formatter. |
|
✅ With the latest revision this PR passed the undef deprecator. |
🐧 Linux x64 Test Results
✅ The build succeeded and all tests passed. |
ff301a9 to
8a49c4f
Compare
8a49c4f to
d6a0f2b
Compare
edf7470 to
41c9213
Compare
🪟 Windows x64 Test Results
✅ The build succeeded and all tests passed. |
e495a98 to
f6d54f9
Compare
9f74882 to
a1c6c6b
Compare
| ; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 | ||
| ; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96 | ||
| ; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] | ||
| ; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer) |
There was a problem hiding this comment.
Hmm, this looks like a bad decision by the loop vectoriser and could lead to regressions. Perhaps the code gets cleaned up in a later pass? I'll play around with this patch for a bit.
There was a problem hiding this comment.
It looks like this isn't a problem for the common case when we're not forcing target costs. For example, for C code like this:
void foo(unsigned char *dst, int n) {
for (int i = 0; i < n; i += 3) {
dst[(i * 3) + 2] = 3;
dst[(i * 3) + 1] = 3;
dst[i * 3] = 3;
}
}
when compiling with clang -O3 -mcpu=neoverse-v1 -S -emit-llvm foo.c we end up replicating instead. What's interesting about this case is that the vectorised form (store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1) may actually be the best, but the cost model still prefers replication.
| @@ -1,9 +1,10 @@ | |||
| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 | |||
There was a problem hiding this comment.
It's very hard to see what's actually changed in this diff. Can you either:
- Pre-commit a NFC patch to automatically generate CHECK lines, then rebase this PR on top? Or,
- Keep the existing style of the test without autogeneration and manually update the CHECK lines in a similar way?
Thanks!
There was a problem hiding this comment.
This comment still not addressed, although I've added a new suggestion for updating the @interleave function in a pre-commit PR.
| ; COST1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 32 | ||
| ; COST1-NEXT: [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], 96 | ||
| ; COST1-NEXT: br i1 [[TMP5]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] | ||
| ; COST1-NEXT: [[INTERLEAVED_VEC:%.*]] = call <vscale x 48 x i8> @llvm.vector.interleave3.nxv48i8(<vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer, <vscale x 16 x i8> zeroinitializer) |
There was a problem hiding this comment.
It looks like this isn't a problem for the common case when we're not forcing target costs. For example, for C code like this:
void foo(unsigned char *dst, int n) {
for (int i = 0; i < n; i += 3) {
dst[(i * 3) + 2] = 3;
dst[(i * 3) + 1] = 3;
dst[i * 3] = 3;
}
}
when compiling with clang -O3 -mcpu=neoverse-v1 -S -emit-llvm foo.c we end up replicating instead. What's interesting about this case is that the vectorised form (store <48 x i8> zeroinitializer, ptr [[TMP3]], align 1) may actually be the best, but the cost model still prefers replication.
| return InstructionCost::getInvalid(); | ||
|
|
||
| if (Factor == 3 && | ||
| DL.getTypeSizeInBits(VecTy).getKnownMinValue() < Factor * 128) |
There was a problem hiding this comment.
This suggests we support things like:
%wide.masked.vec = call <vscale x 96 x i8> @llvm.masked.load.nxv96i8(ptr %p, i32 1, <vscale x 96 x i1> %interleaved.mask, <vscale x 96 x i8> poison)
%deinterleaved.vec = call { <vscale x 32 x i8>, <vscale x 32 x i8>, <vscale x 32 x i8> } @llvm.vector.deinterleave3.nxv96i8(<vscale x 96 x i8> %wide.masked.vec)
However, I don't think the loop vectoriser will generate code like that. Perhaps for now it's worth being more precise, i.e.
if (Factor == 3 &&
DL.getTypeSizeInBits(VecTy).getKnownMinValue() != (Factor * 128))
return InstructionCost::getInvalid();
| } else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue, | ||
| ValueInterleaveOps)) { | ||
| return SDValue(); | ||
| // A vector.interleave3 defined in another basic block is unknown here |
There was a problem hiding this comment.
Why is this only a problem for factor 3 interleaves and not 2 or 4?
There was a problem hiding this comment.
Across blocks, we cannot see the original DAG node. I believe for 2 and 4 we can use the power-of-two to identify if it is compatible. We do not have that with factor of 3, so I have had to implement deinterleaveInterleavedValueForSVESt3 as a way to help with that.
Additionally, it is why there is a test specific to factor-3.
There was a problem hiding this comment.
I'd prefer not to include deinterleaveInterleavedValueForSVESt3 in this PR then as it complicates things. I personally think it makes sense for this initial PR to be purely about adding the ability to lower simple cases to ld3/st3. Then a follow-on PR can add deinterleaveInterleavedValueForSVESt3 and be reviewed separately.
| ret void | ||
| } | ||
|
|
||
| ; This test is specific to interleaves of factor 3. |
There was a problem hiding this comment.
Why specific to factor 3? I don't see anything in the IR that's specifically a problem for factor 3.
There was a problem hiding this comment.
If this is really problem then perhaps we should be adding similar tests for factors 2 and 4?
|
Sorry for the delay, there is a release that we were prioritising first. Can we make sure that i1 interleaves / deinterleaves work first, if they might be generated by the vectorizer: |
There was an unsafe PR for i1 deinterleaves/interleaves here: #209231. |
That was something a little different. This is for standard i1 intrinsics, not deinterleaving-loads / stores. They might need to be lowered to stack loads/stores still, but should do so in a way that uses valid types not attempt to store the i1 vectors directly. |
3e02970 to
b6775d6
Compare
|
After some investigating, I believe there will be some potential unoptimal code generation regressions for enabling LD3/ST3 in the loop vectorizer. |
| ; CHECK-NEXT: addvl sp, sp, #3 | ||
| ; CHECK-NEXT: ldr x29, [sp], #16 // 8-byte Folded Reload | ||
| ; CHECK-NEXT: ret | ||
| %retval = call {<vscale x 8 x i1>, <vscale x 8 x i1>, <vscale x 8 x i1>} @llvm.vector.deinterleave3.nxv24i1(<vscale x 24 x i1> %vec) |
There was a problem hiding this comment.
Previously, we were unable to generate tests interleave/deinterleave for values of i1.
bdb3c40 to
10bd8e3
Compare
david-arm
left a comment
david-arm
left a comment
There was a problem hiding this comment.
Hi, I think this PR needs splitting up into at least two PRs. I think we should sort out all the codegen issues first before enabling loop vectorisation of factor-3 interleaving loads and stores. Roughly I think it needs to be something like:
- Add the changes in performInterleavedStoreCombine and performVectorDeinterleaveCombine for the simple load/store without the specialisations added in
deinterleaveInterleavedValueForSVESt3. - Add the specialisations added in
deinterleaveInterleavedValueForSVESt3. I'd also like to understand more thoroughly why we don't need similar tests and logic for factor-2 and factor-4. - Once the backend can support everything the loop vectoriser can throw at it, then we can fix the cost model.
At the moment the PR is difficult to review and assess because there is quite a lot going on. Also, if there is any kind of regression post-commit you'll need to revert everything including the codegen changes, which would be a shame.
| ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF | ||
| ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF | ||
| ; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -S -mcpu=neoverse-v1 -sve-tail-folding=default+reductions+recurrences+reverse | FileCheck %s -check-prefix=CHECK-TF-DEFAULT |
There was a problem hiding this comment.
Hmm, this isn't really the tail-folding default because it also includes reductions+recurrences+reverse loops. I think this should be the same as this RUN line:
; RUN: opt < %s -passes=loop-vectorize -sve-tail-folding-insn-threshold=0 -sve-tail-folding=simple+reductions+recurrences+reverse -S | FileCheck %s -check-prefix=CHECK-TF
| ; CHECK-NEOVERSE-V1: %[[LOAD:.*]] = load <8 x float>, ptr | ||
| ; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6> | ||
| ; CHECK-NEOVERSE-V1: %{{.*}} = shufflevector <8 x float> %[[LOAD]], <8 x float> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7> | ||
| ; CHECK-NOTF-LABEL: define void @interleave( |
There was a problem hiding this comment.
I think the CHECK lines prior to this PR were much nicer and I'd prefer not to create thousands more CHECK lines when we only really care about a few specific things. It looks like the only reason you've had to change this file is because the @interleave function is using factor-3 interleaves that are affected by this PR. I think it would be better to have a separate pre-commit PR that changes this specific test to use factor-4 or factor-2 interleaved accesses. We'd still be testing that tail-folding wasn't enabled by default, but it would then mean the test is unaffected by this PR.
| @@ -1,9 +1,10 @@ | |||
| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 | |||
There was a problem hiding this comment.
This comment still not addressed, although I've added a new suggestion for updating the @interleave function in a pre-commit PR.
| } else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue, | ||
| ValueInterleaveOps)) { | ||
| return SDValue(); | ||
| // A vector.interleave3 defined in another basic block is unknown here |
There was a problem hiding this comment.
I'd prefer not to include deinterleaveInterleavedValueForSVESt3 in this PR then as it complicates things. I personally think it makes sense for this initial PR to be purely about adding the ability to lower simple cases to ld3/st3. Then a follow-on PR can add deinterleaveInterleavedValueForSVESt3 and be reviewed separately.
| ret void | ||
| } | ||
|
|
||
| ; This test is specific to interleaves of factor 3. |
There was a problem hiding this comment.
If this is really problem then perhaps we should be adding similar tests for factors 2 and 4?
| DAG.getMemBasePlusOffset(StackPtr, PackedVT.getStoreSize() * I, DL); | ||
| SDValue L = DAG.getLoad(PackedVT, DL, Chain, Ptr, MachinePointerInfo()); | ||
| Results.push_back(getSVESafeBitCast(OpVT, L, DAG)); | ||
| Results.push_back(IsPredicate |
There was a problem hiding this comment.
This looks more complicated than it should be. I was expecting this PR to be very simple and just a case of reusing the existing lowering code for factors 2 and 4. Why do we need the IsPredicate check?
10bd8e3 to
454dfa0
Compare
Harry-Ramsey
changed the title
b4460d1 to
64db40e
Compare
| SDLoc DL(Op); | ||
| EVT OpVT = Op.getValueType(); | ||
|
|
||
| if (OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 && |
There was a problem hiding this comment.
It might be neater to fold this into the existing
if (OpVT.isScalableVector() && Op->getNumOperands() == 3) {
block below, i.e.
if (OpVT.isScalableVector() && Op->getNumOperands() == 3) {
if (OpVT.getVectorElementType() == MVT::i1)
return lowerPredicateInterleaveWithPackedVectors(Op, DAG);
| SDLoc DL(Op); | ||
| EVT OpVT = Op.getValueType(); | ||
|
|
||
| if (OpVT.isScalableVector() && OpVT.getVectorElementType() == MVT::i1 && |
There was a problem hiding this comment.
Same comment here as in LowerVECTOR_DEINTERLEAVE.
| for (SDValue V : Op->ops()) | ||
| PackedOps.push_back(DAG.getNode(ISD::ZERO_EXTEND, DL, PackedVT, V)); | ||
|
|
||
| SmallVector<EVT, 3> PackedVTs(Op->getNumValues(), PackedVT); |
There was a problem hiding this comment.
I think you can just 3 instead of Op->getNumValues() because num values should equal num operands according to the documentation for the nodes.
|
|
||
| SDValue Zero = DAG.getConstant(0, DL, PackedVT); | ||
| SmallVector<SDValue, 3> Results; | ||
| for (unsigned I = 0; I != Op->getNumValues(); ++I) |
There was a problem hiding this comment.
nit: You can replace Op->getNumValues() with 3 here I think
| ret { <vscale x 2 x double>, <vscale x 2 x double> } %deinterleaved.vec | ||
| } | ||
|
|
||
| define { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @foo_ld3_nxv16i8(<vscale x 16 x i1> %mask, ptr %p) { |
There was a problem hiding this comment.
It would be good to have variants for other element sizes too, similar to the ld4 case below.
| ret void | ||
| } | ||
|
|
||
| define void @foo_st3_nxv16i8(<vscale x 16 x i1> %mask, <vscale x 16 x i8> %val1, <vscale x 16 x i8> %val2, <vscale x 16 x i8> %val3, ptr %p) { |
There was a problem hiding this comment.
Would be good to have variants for other element sizes too similar to the st4 cases below.
| ; CHECK-NEXT: cmpne p1.h, p0/z, z0.h, #0 | ||
| ; CHECK-NEXT: cmpne p2.h, p0/z, z1.h, #0 | ||
| ; CHECK-NEXT: cmpne p3.h, p0/z, z2.h, #0 | ||
| ; CHECK-NEXT: uzp1 p0.b, p2.b, p1.b |
There was a problem hiding this comment.
This confused me at first until I realised the <vscale x 24 x i1> type has been widened to <vscale x 32 x i1> and the <vscale x 8 x i1> predicates have been packed into it. This might be ok, but it would be good to have other variants tested too to make sure they're all correct, e.g. @llvm.vector.interleave3.nxv12i1 and @llvm.vector.interleave3.nxv6i1
| ; CHECK-NEXT: ldr z0, [sp] | ||
| ; CHECK-NEXT: ldr z1, [sp, #1, mul vl] | ||
| ; CHECK-NEXT: ldr z2, [sp, #2, mul vl] | ||
| ; CHECK-NEXT: cmpne p0.b, p2/z, z0.b, #0 |
There was a problem hiding this comment.
This looks better because the 3 parts are naturally packed into 3 registers that make up <vscale x 48 x i1>
| } | ||
| } | ||
|
|
||
| static SDValue lowerPredicateInterleaveWithPackedVectors(SDValue Op, |
There was a problem hiding this comment.
This should probably be a separate PR, in case there are independent problems
Enable factor-3 scalable vector interleave and deinterleave lowering for SVE, including the corresponding interleaved load/store combines. Factor-3 operations are lowered through the existing LD3/ST3-based path, which operates on packed data vectors. Unlike factors 2 and 4, SVE has no ZIP/UZP instruction sequence specifically ror factor-3 operations. For LD3/ST3 predicate operands must be promoted into Z-register data vectors before lowering, then convert the results back to predicates by comparing against zero. This avoids invalid predicate uses of getSVESafeBitCast.
64db40e to
f00460c
Compare
3 participants
This commit enables LoopVectorize to lower scalable vector.[de]interleave3 to use SVE ld3/st3 instructions.