Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@ target triple = "x86_64-unknown-linux-gnu"
;
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 17 for VF 2 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 49 for VF 4 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 98 for VF 8 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 228 for VF 16 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 456 for VF 32 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 35 for VF 4 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 70 for VF 8 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 172 for VF 16 For instruction: store i16 %v3, i16* %out3, align 2
; AVX1: LV: Found an estimated cost of 344 for VF 32 For instruction: store i16 %v3, i16* %out3, align 2
;
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v3, i16* %out3, align 2
; AVX2: LV: Found an estimated cost of 3 for VF 2 For instruction: store i16 %v3, i16* %out3, align 2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,16 +18,16 @@ target triple = "x86_64-unknown-linux-gnu"
; SSE2: LV: Found an estimated cost of 85 for VF 8 For instruction: store i16 %v4, i16* %out4, align 2
;
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 28 for VF 2 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 58 for VF 4 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 115 for VF 8 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 285 for VF 16 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 27 for VF 2 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 45 for VF 4 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 88 for VF 8 For instruction: store i16 %v4, i16* %out4, align 2
; AVX1: LV: Found an estimated cost of 215 for VF 16 For instruction: store i16 %v4, i16* %out4, align 2
;
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 28 for VF 2 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 58 for VF 4 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 115 for VF 8 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 285 for VF 16 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 27 for VF 2 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 45 for VF 4 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 88 for VF 8 For instruction: store i16 %v4, i16* %out4, align 2
; AVX2: LV: Found an estimated cost of 215 for VF 16 For instruction: store i16 %v4, i16* %out4, align 2
;
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v4, i16* %out4, align 2
; AVX512: LV: Found an estimated cost of 11 for VF 2 For instruction: store i16 %v4, i16* %out4, align 2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ target triple = "x86_64-unknown-linux-gnu"
; SSE2: LV: Found an estimated cost of 102 for VF 8 For instruction: store i16 %v5, i16* %out5, align 2
;
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 35 for VF 2 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 66 for VF 4 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 147 for VF 8 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 342 for VF 16 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 30 for VF 2 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 53 for VF 4 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 105 for VF 8 For instruction: store i16 %v5, i16* %out5, align 2
; AVX1: LV: Found an estimated cost of 258 for VF 16 For instruction: store i16 %v5, i16* %out5, align 2
;
; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i16 %v5, i16* %out5, align 2
; AVX2: LV: Found an estimated cost of 13 for VF 2 For instruction: store i16 %v5, i16* %out5, align 2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,24 +13,24 @@ target triple = "x86_64-unknown-linux-gnu"
; CHECK: LV: Checking a loop in "test"
;
; SSE2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 27 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 79 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 238 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 478 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 14 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 30 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 62 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1
; SSE2: LV: Found an estimated cost of 126 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1
;
; AVX1: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 9 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 17 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 33 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 97 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 226 for VF 32 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 67 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1
; AVX1: LV: Found an estimated cost of 166 for VF 32 For instruction: store i8 %v1, i8* %out1, align 1

; AVX2: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 9 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 17 for VF 4 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 33 for VF 8 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 97 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 226 for VF 32 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 67 for VF 16 For instruction: store i8 %v1, i8* %out1, align 1
; AVX2: LV: Found an estimated cost of 166 for VF 32 For instruction: store i8 %v1, i8* %out1, align 1
;
; AVX512: LV: Found an estimated cost of 1 for VF 1 For instruction: store i8 %v1, i8* %out1, align 1
; AVX512: LV: Found an estimated cost of 4 for VF 2 For instruction: store i8 %v1, i8* %out1, align 1
Expand Down
89 changes: 59 additions & 30 deletions llvm/test/Transforms/LoopVectorize/X86/interleaving.ll
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine < %s | FileCheck %s --check-prefix=SSE
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=sandybridge < %s | FileCheck %s --check-prefix=AVX
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=haswell < %s | FileCheck %s --check-prefix=AVX
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=sandybridge < %s | FileCheck %s --check-prefix=AVX1
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=haswell < %s | FileCheck %s --check-prefix=AVX2
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=slm < %s | FileCheck %s --check-prefix=SSE
; RUN: opt -S -mtriple=x86_64-pc_linux -loop-vectorize -instcombine -mcpu=atom < %s | FileCheck %s --check-prefix=SSE

Expand All @@ -26,34 +26,63 @@ define void @foo(i32* noalias nocapture %a, i32* noalias nocapture readonly %b)
; SSE-NEXT: [[EXITCOND:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], 1024
; SSE-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
;
; AVX-LABEL: @foo(
; AVX-NEXT: entry:
; AVX-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; AVX: vector.ph:
; AVX-NEXT: br label [[VECTOR_BODY:%.*]]
; AVX: vector.body:
; AVX-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; AVX-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; AVX-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
; AVX-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <8 x i32>*
; AVX-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
; AVX-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; AVX-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; AVX-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; AVX-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
; AVX-NEXT: [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
; AVX-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* [[TMP5]], align 4
; AVX-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; AVX-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; AVX-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], [[LOOP0:!llvm.loop !.*]]
; AVX: middle.block:
; AVX-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
; AVX: scalar.ph:
; AVX-NEXT: br label [[FOR_BODY:%.*]]
; AVX: for.cond.cleanup:
; AVX-NEXT: ret void
; AVX: for.body:
; AVX-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], [[LOOP2:!llvm.loop !.*]]
; AVX1-LABEL: @foo(
; AVX1-NEXT: entry:
; AVX1-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; AVX1: vector.ph:
; AVX1-NEXT: br label [[VECTOR_BODY:%.*]]
; AVX1: vector.body:
; AVX1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; AVX1-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; AVX1-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
; AVX1-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <8 x i32>*
; AVX1-NEXT: [[WIDE_VEC:%.*]] = load <8 x i32>, <8 x i32>* [[TMP2]], align 4
; AVX1-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 0, i32 2, i32 4, i32 6>
; AVX1-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <8 x i32> [[WIDE_VEC]], <8 x i32> poison, <4 x i32> <i32 1, i32 3, i32 5, i32 7>
; AVX1-NEXT: [[TMP3:%.*]] = add nsw <4 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; AVX1-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
; AVX1-NEXT: [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <4 x i32>*
; AVX1-NEXT: store <4 x i32> [[TMP3]], <4 x i32>* [[TMP5]], align 4
; AVX1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4
; AVX1-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; AVX1-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; AVX1: middle.block:
; AVX1-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
; AVX1: scalar.ph:
; AVX1-NEXT: br label [[FOR_BODY:%.*]]
; AVX1: for.cond.cleanup:
; AVX1-NEXT: ret void
; AVX1: for.body:
; AVX1-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
;
; AVX2-LABEL: @foo(
; AVX2-NEXT: entry:
; AVX2-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
; AVX2: vector.ph:
; AVX2-NEXT: br label [[VECTOR_BODY:%.*]]
; AVX2: vector.body:
; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
; AVX2-NEXT: [[TMP0:%.*]] = shl nsw i64 [[INDEX]], 1
; AVX2-NEXT: [[TMP1:%.*]] = getelementptr inbounds i32, i32* [[B:%.*]], i64 [[TMP0]]
; AVX2-NEXT: [[TMP2:%.*]] = bitcast i32* [[TMP1]] to <16 x i32>*
; AVX2-NEXT: [[WIDE_VEC:%.*]] = load <16 x i32>, <16 x i32>* [[TMP2]], align 4
; AVX2-NEXT: [[STRIDED_VEC:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
; AVX2-NEXT: [[STRIDED_VEC1:%.*]] = shufflevector <16 x i32> [[WIDE_VEC]], <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
; AVX2-NEXT: [[TMP3:%.*]] = add nsw <8 x i32> [[STRIDED_VEC1]], [[STRIDED_VEC]]
; AVX2-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, i32* [[A:%.*]], i64 [[INDEX]]
; AVX2-NEXT: [[TMP5:%.*]] = bitcast i32* [[TMP4]] to <8 x i32>*
; AVX2-NEXT: store <8 x i32> [[TMP3]], <8 x i32>* [[TMP5]], align 4
; AVX2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 8
; AVX2-NEXT: [[TMP6:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
; AVX2-NEXT: br i1 [[TMP6]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
; AVX2: middle.block:
; AVX2-NEXT: br i1 true, label [[FOR_COND_CLEANUP:%.*]], label [[SCALAR_PH]]
; AVX2: scalar.ph:
; AVX2-NEXT: br label [[FOR_BODY:%.*]]
; AVX2: for.cond.cleanup:
; AVX2-NEXT: ret void
; AVX2: for.body:
; AVX2-NEXT: br i1 undef, label [[FOR_COND_CLEANUP]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]]
;
entry:
br label %for.body
Expand Down