From 5ddd3557086da3033540441b45749908751049a5 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 24 Nov 2025 15:11:40 +0000 Subject: [PATCH 1/2] [VPlan] Handle scalar VPWidenPointerInduction in convertToConcreteRecipes. In some case, VPWidenPointerInductions become only used by scalars after legalizeAndOptimizationInducftions was already run, for example due to some VPlan optimizations. Move the code to scalarize VPWidenPointerInductions to a helper and use it if needed. This fixes a crash after #148274 in the added test case. --- .../Transforms/Vectorize/VPlanTransforms.cpp | 38 +++++-- .../LoopVectorize/RISCV/pointer-induction.ll | 102 ++++++++++++++++++ 2 files changed, 130 insertions(+), 10 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e7a8773be067b..5079fa488ec46 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -668,6 +668,23 @@ static SmallVector collectUsersRecursively(VPValue *V) { return Users.takeVector(); } +/// Scalarize a VPWidenPointerInductionRecipe by replacing it with a PtrAdd +/// (IndStart, ScalarIVSteps (0, Step)). This is used when the recipe only +/// generates scalar values. +static VPValue * +scalarizeVPWidenPointerInduction(VPWidenPointerInductionRecipe *PtrIV, + VPlan &Plan, VPBuilder &Builder) { + const InductionDescriptor &ID = PtrIV->getInductionDescriptor(); + VPValue *StartV = Plan.getConstantInt(ID.getStep()->getType(), 0); + VPValue *StepV = PtrIV->getOperand(1); + VPScalarIVStepsRecipe *Steps = createScalarIVSteps( + Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr, + nullptr, StartV, StepV, PtrIV->getDebugLoc(), Builder); + + return Builder.createPtrAdd(PtrIV->getStartValue(), Steps, + PtrIV->getDebugLoc(), "next.gep"); +} + /// Legalize VPWidenPointerInductionRecipe, by replacing it with a PtrAdd /// (IndStart, ScalarIVSteps (0, Step)) if only its scalar values are used, as /// VPWidenPointerInductionRecipe will generate vectors only. If some users @@ -720,16 +737,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { if (!PtrIV->onlyScalarsGenerated(Plan.hasScalableVF())) continue; - const InductionDescriptor &ID = PtrIV->getInductionDescriptor(); - VPValue *StartV = Plan.getConstantInt(ID.getStep()->getType(), 0); - VPValue *StepV = PtrIV->getOperand(1); - VPScalarIVStepsRecipe *Steps = createScalarIVSteps( - Plan, InductionDescriptor::IK_IntInduction, Instruction::Add, nullptr, - nullptr, StartV, StepV, PtrIV->getDebugLoc(), Builder); - - VPValue *PtrAdd = Builder.createPtrAdd(PtrIV->getStartValue(), Steps, - PtrIV->getDebugLoc(), "next.gep"); - + VPValue *PtrAdd = scalarizeVPWidenPointerInduction(PtrIV, Plan, Builder); PtrIV->replaceAllUsesWith(PtrAdd); continue; } @@ -3502,6 +3510,16 @@ void VPlanTransforms::convertToConcreteRecipes(VPlan &Plan) { } if (auto *WidenIVR = dyn_cast(&R)) { + // If the recipe only generates scalars, scalarize it instead of + // expanding it. + if (WidenIVR->onlyScalarsGenerated(Plan.hasScalableVF())) { + VPBuilder Builder(WidenIVR); + VPValue *PtrAdd = + scalarizeVPWidenPointerInduction(WidenIVR, Plan, Builder); + WidenIVR->replaceAllUsesWith(PtrAdd); + ToRemove.push_back(WidenIVR); + continue; + } expandVPWidenPointerInduction(WidenIVR, TypeInfo); ToRemove.push_back(WidenIVR); continue; diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll index fa710cb8d65b1..47bb6b469c8ad 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll @@ -61,4 +61,106 @@ exit: ret void } +define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c) #1 { +; CHECK-LABEL: define i1 @scalarize_ptr_induction( +; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]], ptr noalias [[DST:%.*]], i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[START5:%.*]] = ptrtoint ptr [[START]] to i64 +; CHECK-NEXT: [[END4:%.*]] = ptrtoint ptr [[END]] to i64 +; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 +; CHECK-NEXT: [[END1:%.*]] = ptrtoint ptr [[END]] to i64 +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[END4]], -12 +; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START5]] +; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP1]], 12 +; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1 +; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP5]], i64 15) +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], [[UMAX]] +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK: [[VECTOR_MEMCHECK]]: +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8 +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[END1]], -12 +; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[TMP6]], [[START2]] +; CHECK-NEXT: [[TMP8:%.*]] = udiv i64 [[TMP7]], 12 +; CHECK-NEXT: [[TMP9:%.*]] = mul nuw i64 [[TMP8]], 12 +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[TMP9]], 8 +; CHECK-NEXT: [[SCEVGEP3:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]] +; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP3]] +; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[START]], [[SCEVGEP]] +; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK: [[VECTOR_PH]]: +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2 +; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP12]] +; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 +; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[N_MOD_VF]] +; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP14]] +; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[N_VEC]], 12 +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]] +; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] +; CHECK: [[VECTOR_BODY]]: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 12 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] +; CHECK-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[NEXT_GEP]], align 4, !alias.scope [[META3:![0-9]+]] +; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { , , } @llvm.vector.deinterleave3.nxv6i32( [[WIDE_VEC]]) +; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { , , } [[STRIDED_VEC]], 0 +; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { , , } [[STRIDED_VEC]], 1 +; CHECK-NEXT: [[TMP19:%.*]] = zext [[TMP18]] to +; CHECK-NEXT: [[TMP20:%.*]] = mul [[TMP19]], splat (i64 -7070675565921424023) +; CHECK-NEXT: [[TMP21:%.*]] = add [[TMP20]], splat (i64 -4) +; CHECK-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 2 +; CHECK-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1 +; CHECK-NEXT: [[TMP25:%.*]] = extractelement [[TMP21]], i32 [[TMP24]] +; CHECK-NEXT: store i64 [[TMP25]], ptr [[DST]], align 1, !alias.scope [[META6:![0-9]+]], !noalias [[META3]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK: [[MIDDLE_BLOCK]]: +; CHECK-NEXT: br label %[[SCALAR_PH]] +; CHECK: [[SCALAR_PH]]: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ], [ [[START]], %[[VECTOR_MEMCHECK]] ] +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 4 +; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4 +; CHECK-NEXT: [[EXT:%.*]] = zext i32 [[L]] to i64 +; CHECK-NEXT: [[UNUSED:%.*]] = load i32, ptr [[PTR_IV]], align 4 +; CHECK-NEXT: [[MUL1:%.*]] = mul i64 [[EXT]], -7070675565921424023 +; CHECK-NEXT: [[MUL2:%.*]] = add i64 [[MUL1]], -4 +; CHECK-NEXT: store i64 [[MUL2]], ptr [[DST]], align 1 +; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 12 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] +; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[CMP]], i1 true, i1 false +; CHECK-NEXT: br i1 [[OR_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ] +; CHECK-NEXT: ret i1 [[CMP_LCSSA]] +; +entry: + br label %loop + +loop: + %ptr.iv = phi ptr [ %start, %entry ], [ %ptr.iv.next, %loop ] + %gep = getelementptr i8, ptr %ptr.iv, i64 4 + %l = load i32, ptr %gep, align 4 + %ext = zext i32 %l to i64 + %unused = load i32, ptr %ptr.iv, align 4 + %mul1 = mul i64 %ext, -7070675565921424023 + %mul2 = add i64 %mul1, -4 + store i64 %mul2, ptr %dst, align 1 + %ptr.iv.next = getelementptr nusw i8, ptr %ptr.iv, i64 12 + %cmp = icmp eq ptr %ptr.iv.next, %end + %or.cond = select i1 %cmp, i1 true, i1 false + br i1 %or.cond, label %exit, label %loop + +exit: + ret i1 %cmp +} + attributes #0 = { "target-features"="+v" } +attributes #1 = { "target-cpu"="sifive-p670" } From 2c151e0e09000333723b94a68fd107709068b667 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Thu, 27 Nov 2025 21:25:21 +0000 Subject: [PATCH 2/2] !fixup update test --- .../LoopVectorize/RISCV/pointer-induction.ll | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll index 47bb6b469c8ad..164a5cd1ae3c0 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/pointer-induction.ll @@ -64,7 +64,7 @@ exit: define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c) #1 { ; CHECK-LABEL: define i1 @scalarize_ptr_induction( ; CHECK-SAME: ptr [[START:%.*]], ptr [[END:%.*]], ptr noalias [[DST:%.*]], i1 [[C:%.*]]) #[[ATTR1:[0-9]+]] { -; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: [[ENTRY:.*:]] ; CHECK-NEXT: [[START5:%.*]] = ptrtoint ptr [[START]] to i64 ; CHECK-NEXT: [[END4:%.*]] = ptrtoint ptr [[END]] to i64 ; CHECK-NEXT: [[START2:%.*]] = ptrtoint ptr [[START]] to i64 @@ -73,11 +73,7 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c ; CHECK-NEXT: [[TMP1:%.*]] = sub i64 [[TMP0]], [[START5]] ; CHECK-NEXT: [[TMP2:%.*]] = udiv i64 [[TMP1]], 12 ; CHECK-NEXT: [[TMP3:%.*]] = add nuw nsw i64 [[TMP2]], 1 -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw i64 [[TMP4]], 1 -; CHECK-NEXT: [[UMAX:%.*]] = call i64 @llvm.umax.i64(i64 [[TMP5]], i64 15) -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ule i64 [[TMP3]], [[UMAX]] -; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label %[[SCALAR_PH:.*]], label %[[VECTOR_MEMCHECK:.*]] +; CHECK-NEXT: br label %[[VECTOR_MEMCHECK:.*]] ; CHECK: [[VECTOR_MEMCHECK]]: ; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[DST]], i64 8 ; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[END1]], -12 @@ -89,43 +85,45 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c ; CHECK-NEXT: [[BOUND0:%.*]] = icmp ult ptr [[DST]], [[SCEVGEP3]] ; CHECK-NEXT: [[BOUND1:%.*]] = icmp ult ptr [[START]], [[SCEVGEP]] ; CHECK-NEXT: [[FOUND_CONFLICT:%.*]] = and i1 [[BOUND0]], [[BOUND1]] -; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH]], label %[[VECTOR_PH:.*]] +; CHECK-NEXT: br i1 [[FOUND_CONFLICT]], label %[[SCALAR_PH:.*]], label %[[VECTOR_PH:.*]] ; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul nuw i64 [[TMP11]], 2 -; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP3]], [[TMP12]] -; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 -; CHECK-NEXT: [[TMP14:%.*]] = select i1 [[TMP13]], i64 [[TMP12]], i64 [[N_MOD_VF]] -; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP3]], [[TMP14]] -; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[N_VEC]], 12 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP15]] +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[DST]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; CHECK-NEXT: [[BROADCAST_SPLATINSERT6:%.*]] = insertelement poison, ptr [[END]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLAT7:%.*]] = shufflevector [[BROADCAST_SPLATINSERT6]], poison, zeroinitializer ; CHECK-NEXT: br label %[[VECTOR_BODY:.*]] ; CHECK: [[VECTOR_BODY]]: -; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ] -; CHECK-NEXT: [[OFFSET_IDX:%.*]] = mul i64 [[INDEX]], 12 -; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[OFFSET_IDX]] -; CHECK-NEXT: [[WIDE_VEC:%.*]] = load , ptr [[NEXT_GEP]], align 4, !alias.scope [[META3:![0-9]+]] -; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { , , } @llvm.vector.deinterleave3.nxv6i32( [[WIDE_VEC]]) -; CHECK-NEXT: [[TMP17:%.*]] = extractvalue { , , } [[STRIDED_VEC]], 0 -; CHECK-NEXT: [[TMP18:%.*]] = extractvalue { , , } [[STRIDED_VEC]], 1 +; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], %[[VECTOR_PH]] ], [ [[PTR_IND:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[AVL:%.*]] = phi i64 [ [[TMP3]], %[[VECTOR_PH]] ], [ [[AVL_NEXT:%.*]], %[[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP13:%.*]] = call @llvm.stepvector.nxv2i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul [[TMP13]], splat (i64 12) +; CHECK-NEXT: [[VECTOR_GEP:%.*]] = getelementptr i8, ptr [[POINTER_PHI]], [[TMP14]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.experimental.get.vector.length.i64(i64 [[AVL]], i32 2, i1 true) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, [[VECTOR_GEP]], i64 4 +; CHECK-NEXT: [[TMP18:%.*]] = call @llvm.vp.gather.nxv2i32.nxv2p0( align 4 [[TMP12]], splat (i1 true), i32 [[TMP11]]), !alias.scope [[META3:![0-9]+]] ; CHECK-NEXT: [[TMP19:%.*]] = zext [[TMP18]] to ; CHECK-NEXT: [[TMP20:%.*]] = mul [[TMP19]], splat (i64 -7070675565921424023) ; CHECK-NEXT: [[TMP21:%.*]] = add [[TMP20]], splat (i64 -4) -; CHECK-NEXT: [[TMP22:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i32 [[TMP22]], 2 -; CHECK-NEXT: [[TMP24:%.*]] = sub i32 [[TMP23]], 1 -; CHECK-NEXT: [[TMP25:%.*]] = extractelement [[TMP21]], i32 [[TMP24]] -; CHECK-NEXT: store i64 [[TMP25]], ptr [[DST]], align 1, !alias.scope [[META6:![0-9]+]], !noalias [[META3]] -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] -; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP26]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] +; CHECK-NEXT: call void @llvm.vp.scatter.nxv2i64.nxv2p0( [[TMP21]], align 1 [[BROADCAST_SPLAT]], splat (i1 true), i32 [[TMP11]]), !alias.scope [[META6:![0-9]+]], !noalias [[META3]] +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr nusw i8, [[VECTOR_GEP]], i64 12 +; CHECK-NEXT: [[TMP17:%.*]] = icmp eq [[TMP16]], [[BROADCAST_SPLAT7]] +; CHECK-NEXT: [[TMP26:%.*]] = zext i32 [[TMP11]] to i64 +; CHECK-NEXT: [[AVL_NEXT]] = sub nuw i64 [[AVL]], [[TMP26]] +; CHECK-NEXT: [[TMP27:%.*]] = mul i64 12, [[TMP26]] +; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP27]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[AVL_NEXT]], 0 +; CHECK-NEXT: br i1 [[TMP28]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: [[MIDDLE_BLOCK]]: -; CHECK-NEXT: br label %[[SCALAR_PH]] +; CHECK-NEXT: [[TMP29:%.*]] = sub i64 [[TMP26]], 1 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul nuw i64 [[TMP22]], 2 +; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 0 +; CHECK-NEXT: [[TMP25:%.*]] = extractelement [[TMP17]], i64 [[TMP29]] +; CHECK-NEXT: br label %[[EXIT:.*]] ; CHECK: [[SCALAR_PH]]: -; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi ptr [ [[TMP16]], %[[MIDDLE_BLOCK]] ], [ [[START]], %[[ENTRY]] ], [ [[START]], %[[VECTOR_MEMCHECK]] ] ; CHECK-NEXT: br label %[[LOOP:.*]] ; CHECK: [[LOOP]]: -; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[BC_RESUME_VAL]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[PTR_IV:%.*]] = phi ptr [ [[START]], %[[SCALAR_PH]] ], [ [[PTR_IV_NEXT:%.*]], %[[LOOP]] ] ; CHECK-NEXT: [[GEP:%.*]] = getelementptr i8, ptr [[PTR_IV]], i64 4 ; CHECK-NEXT: [[L:%.*]] = load i32, ptr [[GEP]], align 4 ; CHECK-NEXT: [[EXT:%.*]] = zext i32 [[L]] to i64 @@ -136,9 +134,9 @@ define i1 @scalarize_ptr_induction(ptr %start, ptr %end, ptr noalias %dst, i1 %c ; CHECK-NEXT: [[PTR_IV_NEXT]] = getelementptr nusw i8, ptr [[PTR_IV]], i64 12 ; CHECK-NEXT: [[CMP:%.*]] = icmp eq ptr [[PTR_IV_NEXT]], [[END]] ; CHECK-NEXT: [[OR_COND:%.*]] = select i1 [[CMP]], i1 true, i1 false -; CHECK-NEXT: br i1 [[OR_COND]], label %[[EXIT:.*]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] +; CHECK-NEXT: br i1 [[OR_COND]], label %[[EXIT]], label %[[LOOP]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: [[EXIT]]: -; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ] +; CHECK-NEXT: [[CMP_LCSSA:%.*]] = phi i1 [ [[CMP]], %[[LOOP]] ], [ [[TMP25]], %[[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret i1 [[CMP_LCSSA]] ; entry: