diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index f460d95f5e652..1054f6aab0748 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8583,43 +8583,6 @@ VPRecipeOrVPValueTy VPRecipeBuilder::handleReplication(Instruction *I, return toVPRecipeResult(Recipe); } -VPRegionBlock * -VPRecipeBuilder::createReplicateRegion(VPReplicateRecipe *PredRecipe, - VPlan &Plan) { - Instruction *Instr = PredRecipe->getUnderlyingInstr(); - // Build the triangular if-then region. - std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str(); - assert(Instr->getParent() && "Predicated instruction not in any basic block"); - auto *BlockInMask = PredRecipe->getMask(); - auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask); - auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe); - - // Replace predicated replicate recipe with a replicate recipe without a - // mask but in the replicate region. - auto *RecipeWithoutMask = new VPReplicateRecipe( - PredRecipe->getUnderlyingInstr(), - make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())), - PredRecipe->isUniform()); - auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask); - - VPPredInstPHIRecipe *PHIRecipe = nullptr; - if (PredRecipe->getNumUsers() != 0) { - PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask); - PredRecipe->replaceAllUsesWith(PHIRecipe); - PHIRecipe->setOperand(0, RecipeWithoutMask); - } - PredRecipe->eraseFromParent(); - auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe); - VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true); - - // Note: first set Entry as region entry and then connect successors starting - // from it in order, to propagate the "parent" of each VPBasicBlock. - VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, Entry); - VPBlockUtils::connectBlocks(Pred, Exiting); - - return Region; -} - VPRecipeOrVPValueTy VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, ArrayRef Operands, @@ -9076,7 +9039,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes( VPlanTransforms::optimizeInductions(*Plan, *PSE.getSE()); VPlanTransforms::removeDeadRecipes(*Plan); - VPlanTransforms::createAndOptimizeReplicateRegions(*Plan, RecipeBuilder); + VPlanTransforms::createAndOptimizeReplicateRegions(*Plan); VPlanTransforms::removeRedundantExpandSCEVRecipes(*Plan); VPlanTransforms::mergeBlocksIntoPredecessors(*Plan); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index ca8ef71a48fbf..8313b2ca4fdbb 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -159,10 +159,6 @@ class VPRecipeBuilder { return Ingredient2Recipe[I]; } - /// Create a replicating region for \p PredRecipe. - VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, - VPlan &Plan); - /// Build a VPReplicationRecipe for \p I. If it is predicated, add the mask as /// last operand. Range.End may be decreased to ensure same recipe behavior /// from \p Range.Start to \p Range.End. diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index e642d264b1c75..99f600002336d 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -314,7 +314,43 @@ static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan) { return !DeletedRegions.empty(); } -static void addReplicateRegions(VPlan &Plan, VPRecipeBuilder &Builder) { +static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, + VPlan &Plan) { + Instruction *Instr = PredRecipe->getUnderlyingInstr(); + // Build the triangular if-then region. + std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str(); + assert(Instr->getParent() && "Predicated instruction not in any basic block"); + auto *BlockInMask = PredRecipe->getMask(); + auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask); + auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe); + + // Replace predicated replicate recipe with a replicate recipe without a + // mask but in the replicate region. + auto *RecipeWithoutMask = new VPReplicateRecipe( + PredRecipe->getUnderlyingInstr(), + make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())), + PredRecipe->isUniform()); + auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask); + + VPPredInstPHIRecipe *PHIRecipe = nullptr; + if (PredRecipe->getNumUsers() != 0) { + PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask); + PredRecipe->replaceAllUsesWith(PHIRecipe); + PHIRecipe->setOperand(0, RecipeWithoutMask); + } + PredRecipe->eraseFromParent(); + auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe); + VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true); + + // Note: first set Entry as region entry and then connect successors starting + // from it in order, to propagate the "parent" of each VPBasicBlock. + VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, Entry); + VPBlockUtils::connectBlocks(Pred, Exiting); + + return Region; +} + +static void addReplicateRegions(VPlan &Plan) { SmallVector WorkList; for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly( vp_depth_first_deep(Plan.getEntry()))) { @@ -334,7 +370,7 @@ static void addReplicateRegions(VPlan &Plan, VPRecipeBuilder &Builder) { SplitBlock->setName( OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : ""); // Record predicated instructions for above packing optimizations. - VPBlockBase *Region = Builder.createReplicateRegion(RepR, Plan); + VPBlockBase *Region = createReplicateRegion(RepR, Plan); Region->setParent(CurrentBlock->getParent()); VPBlockUtils::disconnectBlocks(CurrentBlock, SplitBlock); VPBlockUtils::connectBlocks(CurrentBlock, Region); @@ -342,10 +378,9 @@ static void addReplicateRegions(VPlan &Plan, VPRecipeBuilder &Builder) { } } -void VPlanTransforms::createAndOptimizeReplicateRegions( - VPlan &Plan, VPRecipeBuilder &Builder) { +void VPlanTransforms::createAndOptimizeReplicateRegions(VPlan &Plan) { // Convert masked VPReplicateRecipes to if-then region blocks. - addReplicateRegions(Plan, Builder); + addReplicateRegions(Plan); bool ShouldSimplify = true; while (ShouldSimplify) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index 4c4a11fa72506..e5bd1a42f8778 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -42,8 +42,7 @@ struct VPlanTransforms { /// region block and remove the mask operand. Optimize the created regions by /// iteratively sinking scalar operands into the region, followed by merging /// regions until no improvements are remaining. - static void createAndOptimizeReplicateRegions(VPlan &Plan, - VPRecipeBuilder &Builder); + static void createAndOptimizeReplicateRegions(VPlan &Plan); /// Remove redundant VPBasicBlocks by merging them into their predecessor if /// the predecessor has a single successor.