diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 2163930b02c1c..28748142ae925 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7898,6 +7898,18 @@ void LoopVectorizationPlanner::buildVPlans(ElementCount MinVF, } } +iterator_range>> +VPRecipeBuilder::mapToVPValues(User::op_range Operands) { + std::function Fn = [this](Value *Op) { + if (auto *I = dyn_cast(Op)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(Op); + }; + return map_range(Operands, Fn); +} + VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { assert(is_contained(predecessors(Dst), Src) && "Invalid edge"); @@ -7922,7 +7934,7 @@ VPValue *VPRecipeBuilder::createEdgeMask(BasicBlock *Src, BasicBlock *Dst) { if (OrigLoop->isLoopExiting(Src)) return EdgeMaskCache[Edge] = SrcMask; - VPValue *EdgeMask = Plan.getVPValueOrAddLiveIn(BI->getCondition()); + VPValue *EdgeMask = getVPValueOrAddLiveIn(BI->getCondition(), Plan); assert(EdgeMask && "No Edge Mask found for condition"); if (BI->getSuccessor(0) != Dst) @@ -8383,7 +8395,7 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(Instruction *I, BlockInMask = getBlockInMask(I->getParent()); } - auto *Recipe = new VPReplicateRecipe(I, Plan.mapToVPValues(I->operands()), + auto *Recipe = new VPReplicateRecipe(I, mapToVPValues(I->operands()), IsUniform, BlockInMask); return Recipe; } @@ -8399,10 +8411,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, if (Phi->getParent() != OrigLoop->getHeader()) return tryToBlend(Phi, Operands); - // Always record recipes for header phis. Later first-order recurrence phis - // can have earlier phis as incoming values. - recordRecipeOf(Phi); - if ((Recipe = tryToOptimizeInductionPHI(Phi, Operands, Range))) return Recipe; @@ -8427,14 +8435,6 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr, PhiRecipe = new VPFirstOrderRecurrencePHIRecipe(Phi, *StartV); } - // Record the incoming value from the backedge, so we can add the incoming - // value from the backedge after all recipes have been created. - auto *Inc = cast( - Phi->getIncomingValueForBlock(OrigLoop->getLoopLatch())); - auto RecipeIter = Ingredient2Recipe.find(Inc); - if (RecipeIter == Ingredient2Recipe.end()) - recordRecipeOf(Inc); - PhisToFix.push_back(PhiRecipe); return PhiRecipe; } @@ -8522,7 +8522,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // Add exit values to \p Plan. VPLiveOuts are added for each LCSSA phi in the // original exit block. static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, - VPlan &Plan) { + VPRecipeBuilder &Builder, VPlan &Plan) { BasicBlock *ExitBB = OrigLoop->getUniqueExitBlock(); BasicBlock *ExitingBB = OrigLoop->getExitingBlock(); // Only handle single-exit loops with unique exit blocks for now. @@ -8533,7 +8533,7 @@ static void addUsersInExitBlock(VPBasicBlock *HeaderVPBB, Loop *OrigLoop, for (PHINode &ExitPhi : ExitBB->phis()) { Value *IncomingValue = ExitPhi.getIncomingValueForBlock(ExitingBB); - VPValue *V = Plan.getVPValueOrAddLiveIn(IncomingValue); + VPValue *V = Builder.getVPValueOrAddLiveIn(IncomingValue, Plan); Plan.addLiveOut(&ExitPhi, V); } } @@ -8603,9 +8603,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { if (!getDecisionAndClampRange(applyIG, Range)) continue; InterleaveGroups.insert(IG); - for (unsigned i = 0; i < IG->getFactor(); i++) - if (Instruction *Member = IG->getMember(i)) - RecipeBuilder.recordRecipeOf(Member); }; // --------------------------------------------------------------------------- @@ -8647,7 +8644,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Operands.push_back(Plan->getVPValueOrAddLiveIn( Phi->getIncomingValueForBlock(OrigLoop->getLoopPreheader()))); } else { - auto OpRange = Plan->mapToVPValues(Instr->operands()); + auto OpRange = RecipeBuilder.mapToVPValues(Instr->operands()); Operands = {OpRange.begin(), OpRange.end()}; } @@ -8662,10 +8659,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { RecipeBuilder.tryToCreateWidenRecipe(Instr, Operands, Range, VPBB); if (!Recipe) Recipe = RecipeBuilder.handleReplication(Instr, Range); - for (auto *Def : Recipe->definedValues()) { - auto *UV = Def->getUnderlyingValue(); - Plan->addVPValue(UV, Def); - } RecipeBuilder.setRecipe(Instr, Recipe); if (isa(Recipe)) { @@ -8697,7 +8690,7 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { // and there is nothing to fix from vector loop; phis should have incoming // from scalar loop only. } else - addUsersInExitBlock(HeaderVPBB, OrigLoop, *Plan); + addUsersInExitBlock(HeaderVPBB, OrigLoop, RecipeBuilder, *Plan); assert(isa(Plan->getVectorLoopRegion()) && !Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() && @@ -8765,10 +8758,6 @@ LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(VFRange &Range) { Plan->getVPValueOrAddLiveIn(StrideV)->replaceAllUsesWith(ConstVPV); } - // From this point onwards, VPlan-to-VPlan transformations may change the plan - // in ways that accessing values using original IR values is incorrect. - Plan->disableValue2VPValue(); - VPlanTransforms::dropPoisonGeneratingRecipes(*Plan, [this](BasicBlock *BB) { return Legal->blockNeedsPredication(BB); }); diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index 29a395c357311..fbb70f333685d 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -52,9 +52,8 @@ class VPRecipeBuilder { EdgeMaskCacheTy EdgeMaskCache; BlockMaskCacheTy BlockMaskCache; - // VPlan-VPlan transformations support: Hold a mapping from ingredients to - // their recipe. To save on memory, only do so for selected ingredients, - // marked by having a nullptr entry in this map. + // VPlan construction support: Hold a mapping from ingredients to + // their recipe. DenseMap Ingredient2Recipe; /// Cross-iteration reduction & first-order recurrence phis for which we need @@ -117,13 +116,10 @@ class VPRecipeBuilder { ArrayRef Operands, VFRange &Range, VPBasicBlock *VPBB); - /// Set the recipe created for given ingredient. This operation is a no-op for - /// ingredients that were not marked using a nullptr entry in the map. + /// Set the recipe created for given ingredient. void setRecipe(Instruction *I, VPRecipeBase *R) { - if (!Ingredient2Recipe.count(I)) - return; - assert(Ingredient2Recipe[I] == nullptr && - "Recipe already set for ingredient"); + assert(!Ingredient2Recipe.contains(I) && + "Cannot reset recipe for instruction."); Ingredient2Recipe[I] = R; } @@ -146,14 +142,6 @@ class VPRecipeBuilder { /// between SRC and DST. VPValue *getEdgeMask(BasicBlock *Src, BasicBlock *Dst) const; - /// Mark given ingredient for recording its recipe once one is created for - /// it. - void recordRecipeOf(Instruction *I) { - assert((!Ingredient2Recipe.count(I) || Ingredient2Recipe[I] == nullptr) && - "Recipe already set for ingredient"); - Ingredient2Recipe[I] = nullptr; - } - /// Return the recipe created for given ingredient. VPRecipeBase *getRecipe(Instruction *I) { assert(Ingredient2Recipe.count(I) && @@ -171,6 +159,19 @@ class VPRecipeBuilder { /// Add the incoming values from the backedge to reduction & first-order /// recurrence cross-iteration phis. void fixHeaderPhis(); + + /// Returns a range mapping the values of the range \p Operands to their + /// corresponding VPValues. + iterator_range>> + mapToVPValues(User::op_range Operands); + + VPValue *getVPValueOrAddLiveIn(Value *V, VPlan &Plan) { + if (auto *I = dyn_cast(V)) { + if (auto *R = Ingredient2Recipe.lookup(I)) + return R->getVPSingleValue(); + } + return Plan.getVPValueOrAddLiveIn(V); + } }; } // end namespace llvm diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index d77c7554d50e4..22173954f7cec 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -2872,10 +2872,6 @@ class VPlan { /// definitions are VPValues that hold a pointer to their underlying IR. SmallVector VPLiveInsToFree; - /// Indicates whether it is safe use the Value2VPValue mapping or if the - /// mapping cannot be used any longer, because it is stale. - bool Value2VPValueEnabled = true; - /// Values used outside the plan. MapVector LiveOuts; @@ -2954,10 +2950,6 @@ class VPlan { /// Returns VF * UF of the vector loop region. VPValue &getVFxUF() { return VFxUF; } - /// Mark the plan to indicate that using Value2VPValue is not safe any - /// longer, because it may be stale. - void disableValue2VPValue() { Value2VPValueEnabled = false; } - void addVF(ElementCount VF) { VFs.insert(VF); } void setVF(ElementCount VF) { @@ -2987,8 +2979,7 @@ class VPlan { void setName(const Twine &newName) { Name = newName.str(); } void addVPValue(Value *V, VPValue *VPV) { - assert((Value2VPValueEnabled || VPV->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(VPV->isLiveIn() && "VPV must be a live-in."); assert(V && "Trying to add a null Value to VPlan"); assert(!Value2VPValue.count(V) && "Value already exists in VPlan"); Value2VPValue[V] = VPV; @@ -2998,8 +2989,8 @@ class VPlan { VPValue *getVPValue(Value *V) { assert(V && "Trying to get the VPValue of a null Value"); assert(Value2VPValue.count(V) && "Value does not exist in VPlan"); - assert((Value2VPValueEnabled || Value2VPValue[V]->isLiveIn()) && - "Value2VPValue mapping may be out of date!"); + assert(Value2VPValue[V]->isLiveIn() && + "Only live-ins should be in mapping"); return Value2VPValue[V]; } @@ -3030,16 +3021,6 @@ class VPlan { LLVM_DUMP_METHOD void dump() const; #endif - /// Returns a range mapping the values the range \p Operands to their - /// corresponding VPValues. - iterator_range>> - mapToVPValues(User::op_range Operands) { - std::function Fn = [this](Value *Op) { - return getVPValueOrAddLiveIn(Op); - }; - return map_range(Operands, Fn); - } - /// Returns the VPRegionBlock of the vector loop. VPRegionBlock *getVectorLoopRegion() { return cast(getEntry()->getSingleSuccessor()); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index a91ccefe4b6d7..00cec11485281 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -48,15 +48,14 @@ void VPlanTransforms::VPInstructionsToVPRecipes( VPRecipeBase *NewRecipe = nullptr; if (auto *VPPhi = dyn_cast(&Ingredient)) { auto *Phi = cast(VPPhi->getUnderlyingValue()); - if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) { - VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); - VPValue *Step = - vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); - NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); - } else { - Plan->addVPValue(Phi, VPPhi); + const auto *II = GetIntOrFpInductionDescriptor(Phi); + if (!II) continue; - } + + VPValue *Start = Plan->getVPValueOrAddLiveIn(II->getStartValue()); + VPValue *Step = + vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE); + NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II); } else { assert(isa(&Ingredient) && "only VPInstructions expected here");