diff --git a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp index 216efaf4b2fe9..dbc2e71c785ee 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp @@ -241,10 +241,12 @@ void VPPredicator::convertPhisToBlends(VPBasicBlock *VPBB) { // be duplications since this is a simple recursive scan, but future // optimizations will clean it up. - if (all_equal(make_filter_range(PhiR->incoming_values(), [](VPValue *V) { - return !match(V, m_Poison()); - }))) { - PhiR->replaceAllUsesWith(PhiR->getIncomingValue(0)); + auto NotPoison = make_filter_range(PhiR->incoming_values(), [](VPValue *V) { + return !match(V, m_Poison()); + }); + if (all_equal(NotPoison)) { + PhiR->replaceAllUsesWith(NotPoison.empty() ? PhiR->getIncomingValue(0) + : *NotPoison.begin()); PhiR->eraseFromParent(); continue; } diff --git a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll index 38bdd782257a5..126846456bb10 100644 --- a/llvm/test/Transforms/LoopVectorize/uniform-blend.ll +++ b/llvm/test/Transforms/LoopVectorize/uniform-blend.ll @@ -291,6 +291,89 @@ exit: ret void } +define void @blend_poison_first_argument(ptr %p, i1 %c) { +; CHECK-LABEL: define void @blend_poison_first_argument( +; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi <4 x i64> [ , %[[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: store <4 x i64> [[VEC_IND]], ptr [[TMP0]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[VEC_IND_NEXT]] = add nuw nsw <4 x i64> [[VEC_IND]], splat (i64 4) +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] + br i1 %c, label %if, label %latch + +if: + br label %latch + +latch: + %blend = phi i64 [ %iv, %loop ], [ poison, %if ] + %gep = getelementptr i64, ptr %p, i64 %iv + store i64 %blend, ptr %gep + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp ult i64 %iv, 31 + br i1 %ec, label %loop, label %exit + +exit: + ret void +} + +define void @blend_all_poison(ptr %p, i1 %c) { +; CHECK-LABEL: define void @blend_all_poison( +; CHECK-SAME: ptr [[P:%.*]], i1 [[C:%.*]]) { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: br label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i64, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: store <4 x i64> poison, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i64 [[INDEX_NEXT]], 32 +; CHECK-NEXT: br i1 [[TMP1]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %latch ] + br i1 %c, label %if, label %latch + +if: + br label %latch + +latch: + %blend = phi i64 [ poison, %loop ], [ poison, %if ] + %gep = getelementptr i64, ptr %p, i64 %iv + store i64 %blend, ptr %gep + %iv.next = add nuw nsw i64 %iv, 1 + %ec = icmp ult i64 %iv, 31 + br i1 %ec, label %loop, label %exit + +exit: + ret void +} ;. ; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} @@ -299,4 +382,6 @@ exit: ; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]], [[META2]]} ; CHECK: [[LOOP4]] = distinct !{[[LOOP4]], [[META1]], [[META2]]} ; CHECK: [[LOOP5]] = distinct !{[[LOOP5]], [[META1]], [[META2]]} +; CHECK: [[LOOP6]] = distinct !{[[LOOP6]], [[META1]], [[META2]]} +; CHECK: [[LOOP7]] = distinct !{[[LOOP7]], [[META1]], [[META2]]} ;.