diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 72858e1265d86..ce538cf2c9dab 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1084,7 +1084,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, OpcodeTy Opcode; /// An optional name that can be used for the generated IR instruction. - const std::string Name; + std::string Name; /// Returns true if we can generate a scalar for the first lane only if /// needed. @@ -1183,6 +1183,8 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, /// Returns the symbolic name assigned to the VPInstruction. StringRef getName() const { return Name; } + + void setName(StringRef NewName) { Name = NewName.str(); } }; /// A specialization of VPInstruction augmenting it with a dedicated result diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 92ff0dcf67927..2a51bcc33cf5e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -533,6 +533,15 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL, Plan.getEntry()->swapSuccessors(); createExtractsForLiveOuts(Plan, MiddleVPBB); + + VPBuilder ScalarPHBuilder(ScalarPH); + for (const auto &[PhiR, ScalarPhiR] : zip_equal( + drop_begin(HeaderVPBB->phis()), Plan.getScalarHeader()->phis())) { + auto *VectorPhiR = cast(&PhiR); + auto *ResumePhiR = ScalarPHBuilder.createScalarPhi( + {VectorPhiR, VectorPhiR->getOperand(0)}, VectorPhiR->getDebugLoc()); + cast(&ScalarPhiR)->addOperand(ResumePhiR); + } } std::unique_ptr diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index bb9eed0e0ddb9..1aa7b6aa0de09 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -4413,9 +4413,10 @@ void VPlanTransforms::addBranchWeightToMiddleTerminator( /// Create and return a ResumePhi for \p WideIV, unless it is truncated. If the /// induction recipe is not canonical, creates a VPDerivedIVRecipe to compute /// the end value of the induction. -static VPInstruction *addResumePhiRecipeForInduction( - VPWidenInductionRecipe *WideIV, VPBuilder &VectorPHBuilder, - VPBuilder &ScalarPHBuilder, VPTypeAnalysis &TypeInfo, VPValue *VectorTC) { +static VPValue *addResumePhiRecipeForInduction(VPWidenInductionRecipe *WideIV, + VPBuilder &VectorPHBuilder, + VPTypeAnalysis &TypeInfo, + VPValue *VectorTC) { auto *WideIntOrFp = dyn_cast(WideIV); // Truncated wide inductions resume from the last lane of their vector value // in the last vector iteration which is handled elsewhere. @@ -4441,9 +4442,7 @@ static VPInstruction *addResumePhiRecipeForInduction( WideIV->getDebugLoc()); } - auto *ResumePhiRecipe = ScalarPHBuilder.createScalarPhi( - {EndValue, Start}, WideIV->getDebugLoc(), "bc.resume.val"); - return ResumePhiRecipe; + return EndValue; } void VPlanTransforms::addScalarResumePhis( @@ -4456,21 +4455,18 @@ void VPlanTransforms::addScalarResumePhis( VPBuilder VectorPHBuilder( cast(VectorRegion->getSinglePredecessor())); VPBuilder MiddleBuilder(MiddleVPBB, MiddleVPBB->getFirstNonPhi()); - VPBuilder ScalarPHBuilder(ScalarPH); - for (VPRecipeBase &ScalarPhiR : Plan.getScalarHeader()->phis()) { - auto *ScalarPhiIRI = cast(&ScalarPhiR); + for (VPRecipeBase &PhiR : Plan.getScalarPreheader()->phis()) { + auto *ResumePhiR = cast(&PhiR); // TODO: Extract final value from induction recipe initially, optimize to // pre-computed end value together in optimizeInductionExitUsers. - auto *VectorPhiR = - cast(Builder.getRecipe(&ScalarPhiIRI->getIRPhi())); + auto *VectorPhiR = cast(ResumePhiR->getOperand(0)); if (auto *WideIVR = dyn_cast(VectorPhiR)) { - if (VPInstruction *ResumePhi = addResumePhiRecipeForInduction( - WideIVR, VectorPHBuilder, ScalarPHBuilder, TypeInfo, - &Plan.getVectorTripCount())) { - assert(isa(ResumePhi) && "Expected a phi"); - IVEndValues[WideIVR] = ResumePhi->getOperand(0); - ScalarPhiIRI->addOperand(ResumePhi); + if (VPValue *ResumeV = addResumePhiRecipeForInduction( + WideIVR, VectorPHBuilder, TypeInfo, &Plan.getVectorTripCount())) { + IVEndValues[WideIVR] = ResumeV; + ResumePhiR->setOperand(0, ResumeV); + ResumePhiR->setName("bc.resume.val"); continue; } // TODO: Also handle truncated inductions here. Computing end-values @@ -4492,10 +4488,8 @@ void VPlanTransforms::addScalarResumePhis( ResumeFromVectorLoop = MiddleBuilder.createNaryOp( VPInstruction::ExtractLastElement, {ResumeFromVectorLoop}, {}, "vector.recur.extract"); - StringRef Name = IsFOR ? "scalar.recur.init" : "bc.merge.rdx"; - auto *ResumePhiR = ScalarPHBuilder.createScalarPhi( - {ResumeFromVectorLoop, VectorPhiR->getStartValue()}, {}, Name); - ScalarPhiIRI->addOperand(ResumePhiR); + ResumePhiR->setName(IsFOR ? "scalar.recur.init" : "bc.merge.rdx"); + ResumePhiR->setOperand(0, ResumeFromVectorLoop); } } diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp index b99d656c5c50f..2cc6fdb96c74e 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -113,12 +113,13 @@ compound=true N0 -> N2 [ label="F"] N1 [label = "scalar.ph:\l" + + " EMIT-SCALAR vp\<%6\> = phi [ ir\<%indvars.iv\>, middle.block ], [ ir\<0\>, ir-bb\ ]\l" + "Successor(s): ir-bb\\l" ] N1 -> N3 [ label=""] N3 [label = "ir-bb\:\l" + - " IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ]\l" + + " IR %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %for.body ] (extra operand: vp\<%6\> from scalar.ph)\l" + " IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %indvars.iv\l" + " IR %l1 = load i32, ptr %arr.idx, align 4\l" + " IR %res = add i32 %l1, 10\l" + @@ -282,12 +283,13 @@ compound=true N0 -> N2 [ label="F"] N1 [label = "scalar.ph:\l" + + " EMIT-SCALAR vp\<%6\> = phi [ ir\<%iv\>, middle.block ], [ ir\<0\>, ir-bb\ ]\l" + "Successor(s): ir-bb\\l" ] N1 -> N3 [ label=""] N3 [label = "ir-bb\:\l" + - " IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ]\l" + + " IR %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.latch ] (extra operand: vp\<%6\> from scalar.ph)\l" + " IR %arr.idx = getelementptr inbounds i32, ptr %A, i64 %iv\l" + " IR %l1 = load i32, ptr %arr.idx, align 4\l" + " IR %c = icmp eq i32 %l1, 0\l" + diff --git a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp index 169114ed6c310..cca779142bce4 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanVerifierTest.cpp @@ -346,6 +346,8 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInScalarHeaderVPIRBB) { Function *F = M.getFunction("f"); BasicBlock *LoopHeader = F->getEntryBlock().getSingleSuccessor(); auto Plan = buildVPlan(LoopHeader); + VPValue *Zero = Plan->getConstantInt(32, 0); + Plan->getScalarHeader()->front().addOperand(Zero); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr(); @@ -387,8 +389,6 @@ TEST_F(VPIRVerifierTest, testVerifyIRPhiInExitVPIRBB) { {HeaderBlock->front().getVPSingleValue()}); DefI->insertBefore(Plan->getMiddleBlock()->getTerminator()); Plan->getExitBlocks()[0]->front().addOperand(DefI); - VPValue *Zero = Plan->getConstantInt(32, 0); - Plan->getScalarHeader()->front().addOperand(Zero); #if GTEST_HAS_STREAM_REDIRECTION ::testing::internal::CaptureStderr();