diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp index 59d9d827ea0ca..f61d315f01c7d 100644 --- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp +++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp @@ -3664,6 +3664,13 @@ X86TTIImpl::getReplicationShuffleCost(Type *EltTy, int ReplicationFactor, if (!ST->hasVBMI()) PromEltTyBits = 32; // promote to i32, AVX512F. break; // AVX512VBMI + case 1: + // There is no support for shuffling i1 elements. We *must* promote. + if (ST->hasBWI()) { + PromEltTyBits = 16; // promote to i16, AVX512BW. + break; + } + return bailout(); default: return bailout(); } diff --git a/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll b/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll index 13c8133b6a08a..6fb11bb50b8f8 100644 --- a/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll +++ b/llvm/test/Analysis/CostModel/X86/shuffle-replication-i1.ll @@ -8,21 +8,8 @@ ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx2 | FileCheck %s --check-prefixes=AVX ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f | FileCheck %s --check-prefixes=AVX512F ; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512dq | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512dq,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512dq,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512dq,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F -; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512dq,+avx512bw,+avx512vbmi | FileCheck %s --check-prefixes=AVX512F - +; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX512BW +; RUN: opt < %s -cost-model -mtriple=x86_64-pc-linux-gnu -analyze -mattr=+avx512f,+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW define void @replication_i1_stride2() nounwind { ; SSE2-LABEL: 'replication_i1_stride2' @@ -101,6 +88,17 @@ define void @replication_i1_stride2() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 192 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <128 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 384 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <256 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride2' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <2 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <4 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <8 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 18 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 36 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <256 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <2 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <4 x i32> @@ -190,6 +188,17 @@ define void @replication_i1_stride3() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 256 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <192 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <384 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride3' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <3 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <6 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <12 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <24 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <48 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <96 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 30 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <192 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 60 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <384 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <3 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <6 x i32> @@ -279,6 +288,17 @@ define void @replication_i1_stride4() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 320 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <256 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 640 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <512 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride4' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <4 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <8 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 34 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <256 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 68 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <512 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <4 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <8 x i32> @@ -368,6 +388,17 @@ define void @replication_i1_stride5() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 384 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <320 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 768 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <640 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride5' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <5 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <10 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <20 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <40 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <80 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 27 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <160 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 54 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <320 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 108 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <640 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <5 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <10 x i32> @@ -457,6 +488,17 @@ define void @replication_i1_stride6() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 448 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <384 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 896 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <768 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride6' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 7 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <6 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <12 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <24 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <48 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 15 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <96 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 29 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <192 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 58 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <384 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 116 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <768 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <6 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <12 x i32> @@ -546,6 +588,17 @@ define void @replication_i1_stride7() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 512 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <448 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 1024 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <896 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride7' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <7 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <14 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <28 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <56 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <112 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 31 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <224 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 62 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <448 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 124 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <896 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <7 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <14 x i32> @@ -635,6 +688,17 @@ define void @replication_i1_stride8() nounwind { ; AVX512F-NEXT: Cost Model: Found an estimated cost of 576 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <512 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 1152 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <1024 x i32> ; AVX512F-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; AVX512BW-LABEL: 'replication_i1_stride8' +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <8 x i32> zeroinitializer +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <16 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 5 for instruction: %vf4 = shufflevector <4 x i1> undef, <4 x i1> poison, <32 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %vf8 = shufflevector <8 x i1> undef, <8 x i1> poison, <64 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 17 for instruction: %vf16 = shufflevector <16 x i1> undef, <16 x i1> poison, <128 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 33 for instruction: %vf32 = shufflevector <32 x i1> undef, <32 x i1> poison, <256 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 66 for instruction: %vf64 = shufflevector <64 x i1> undef, <64 x i1> poison, <512 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 132 for instruction: %vf128 = shufflevector <128 x i1> undef, <128 x i1> poison, <1024 x i32> +; AVX512BW-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void ; %vf1 = shufflevector <1 x i1> undef, <1 x i1> poison, <8 x i32> zeroinitializer %vf2 = shufflevector <2 x i1> undef, <2 x i1> poison, <16 x i32>