diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index 490f6391c15a0..29bfd9eee785e 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -5750,11 +5750,14 @@ AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp); bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector; - // A subvector extract can be implemented with an ext (or trivial extract, if - // from lane 0). This currently only handles low or high extracts to prevent - // SLP vectorizer regressions. + // A subvector extract can be implemented with a NEON/SVE ext (or trivial + // extract, if from lane 0) for 128-bit NEON vectors or legal SVE vectors. + // This currently only handles low or high extracts to prevent SLP vectorizer + // regressions. + // Note that SVE's ext instruction is destructive, but it can be fused with + // a movprfx to act like a constructive instruction. if (IsExtractSubvector && LT.second.isFixedLengthVector()) { - if (LT.second.is128BitVector() && + if (LT.second.getFixedSizeInBits() >= 128 && cast(SubTp)->getNumElements() == LT.second.getVectorNumElements() / 2) { if (Index == 0) diff --git a/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll b/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll index 867c35ab79446..07764fbf4acf3 100644 --- a/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll +++ b/llvm/test/Analysis/CostModel/AArch64/shuffle-extract.ll @@ -1,6 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py ; RUN: opt < %s -mtriple=aarch64--linux-gnu -passes="print" -cost-kind=all 2>&1 -disable-output | FileCheck %s +; This tests the cost of fixed-length subvector extracts for NEON. +; For the SVE equivalent test, see sve-vls-shuffle-extract.ll + target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128" define void @extract_half() { diff --git a/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll b/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll new file mode 100644 index 0000000000000..65261a8b2c6d6 --- /dev/null +++ b/llvm/test/Analysis/CostModel/AArch64/sve-vls-shuffle-extract.ll @@ -0,0 +1,339 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py +; RUN: opt < %s -mtriple=aarch64 -mattr=+sve -passes="print" 2>&1 -disable-output \ +; RUN: -aarch64-sve-vector-bits-min=128 \ +; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-1 +; RUN: opt < %s -mtriple=aarch64 -mattr=+sve -passes="print" 2>&1 -disable-output \ +; RUN: -aarch64-sve-vector-bits-min=128 -aarch64-sve-vector-bits-max=128 \ +; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-1 +; RUN: opt < %s -mtriple=aarch64 -mattr=+sve -passes="print" 2>&1 -disable-output \ +; RUN: -aarch64-sve-vector-bits-min=256 \ +; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-2 +; RUN: opt < %s -mtriple=aarch64 -mattr=+sve -passes="print" 2>&1 -disable-output \ +; RUN: -aarch64-sve-vector-bits-min=256 -aarch64-sve-vector-bits-max=256 \ +; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-2 +; RUN: opt < %s -mtriple=aarch64 -mattr=+sve -passes="print" 2>&1 -disable-output \ +; RUN: -aarch64-sve-vector-bits-min=512 \ +; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-4 +; RUN: opt < %s -mtriple=aarch64 -mattr=+sve -passes="print" 2>&1 -disable-output \ +; RUN: -aarch64-sve-vector-bits-min=512 -aarch64-sve-vector-bits-max=512 \ +; RUN: | FileCheck %s --check-prefixes=VSCALE-ANY,VSCALE-4 + +; This tests the cost of fixed-length subvector extracts for SVE, +; either for a minimum vscale or a fixed vscale (aka VLS). +; For the NEON equivalent test, see shuffle-extract.ll + +define void @extract_half_lo() { +; VSCALE-ANY-LABEL: 'extract_half_lo' +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v16i8_lo = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v32i8_lo = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v64i8_lo = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v8i16_lo = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v16i16_lo = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v32i16_lo = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4i32_lo = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v8i32_lo = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v16i32_lo = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v2i64_lo = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> zeroinitializer +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4i64_lo = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v8i64_lo = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v16i8_lo = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> + %v32i8_lo = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> + %v64i8_lo = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> + + %v8i16_lo = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> + %v16i16_lo = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> + %v32i16_lo = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> + + %v4i32_lo = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> + %v8i32_lo = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> + %v16i32_lo = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> + + %v2i64_lo = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> + %v4i64_lo = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> + %v8i64_lo = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> + + ret void +} + +define void @extract_half_hi() { +; VSCALE-1-LABEL: 'extract_half_hi' +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-2-LABEL: 'extract_half_hi' +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-4-LABEL: 'extract_half_hi' +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> + %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> + %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> + + %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> + %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> + %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> + + %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> + %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> + %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> + + %v2i64_hi = shufflevector <2 x i64> poison, <2 x i64> poison, <1 x i32> + %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> + %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> + + ret void +} + +define void @extract_half_unaligned() { +; VSCALE-1-LABEL: 'extract_half_unaligned' +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-2-LABEL: 'extract_half_unaligned' +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-4-LABEL: 'extract_half_unaligned' +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <8 x i32> + %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <16 x i32> + %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <32 x i32> + + %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <4 x i32> + %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <8 x i32> + %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <16 x i32> + + %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <2 x i32> + %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <4 x i32> + %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <8 x i32> + + %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <2 x i32> + %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <4 x i32> + + ret void +} + +define void @extract_qtr_lo() { +; VSCALE-ANY-LABEL: 'extract_qtr_lo' +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v16i8_lo = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v32i8_lo = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v64i8_lo = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v8i16_lo = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v16i16_lo = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v32i16_lo = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4i32_lo = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> zeroinitializer +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v8i32_lo = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v16i32_lo = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v4i64_lo = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> zeroinitializer +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: %v8i64_lo = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-ANY-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v16i8_lo = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> + %v32i8_lo = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> + %v64i8_lo = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> + + %v8i16_lo = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> + %v16i16_lo = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> + %v32i16_lo = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> + + %v4i32_lo = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> + %v8i32_lo = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> + %v16i32_lo = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> + + %v4i64_lo = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> + %v8i64_lo = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> + + ret void +} + +define void @extract_qtr_hi() { +; VSCALE-1-LABEL: 'extract_qtr_hi' +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-2-LABEL: 'extract_qtr_hi' +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-4-LABEL: 'extract_qtr_hi' +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v16i8_hi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> + %v32i8_hi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> + %v64i8_hi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> + + %v8i16_hi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> + %v16i16_hi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> + %v32i16_hi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> + + %v4i32_hi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> + %v8i32_hi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> + %v16i32_hi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> + + %v4i64_hi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> + %v8i64_hi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> + + ret void +} + +define void @extract_qtr_unaligned() { +; VSCALE-1-LABEL: 'extract_qtr_unaligned' +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-1-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-2-LABEL: 'extract_qtr_unaligned' +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-2-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; +; VSCALE-4-LABEL: 'extract_qtr_unaligned' +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> +; VSCALE-4-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret void +; + %v16i8_mi = shufflevector <16 x i8> poison, <16 x i8> poison, <4 x i32> + %v32i8_mi = shufflevector <32 x i8> poison, <32 x i8> poison, <8 x i32> + %v64i8_mi = shufflevector <64 x i8> poison, <64 x i8> poison, <16 x i32> + + %v8i16_mi = shufflevector <8 x i16> poison, <8 x i16> poison, <2 x i32> + %v16i16_mi = shufflevector <16 x i16> poison, <16 x i16> poison, <4 x i32> + %v32i16_mi = shufflevector <32 x i16> poison, <32 x i16> poison, <8 x i32> + + %v4i32_mi = shufflevector <4 x i32> poison, <4 x i32> poison, <1 x i32> + %v8i32_mi = shufflevector <8 x i32> poison, <8 x i32> poison, <2 x i32> + %v16i32_mi = shufflevector <16 x i32> poison, <16 x i32> poison, <4 x i32> + + %v4i64_mi = shufflevector <4 x i64> poison, <4 x i64> poison, <1 x i32> + %v8i64_mi = shufflevector <8 x i64> poison, <8 x i64> poison, <2 x i32> + + ret void +}