diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index b3ba38f6c630e..018c2d21bf46f 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -757,6 +757,31 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { if (!PhiR) continue; + // Try to narrow wide and replicating recipes to uniform recipes, based on + // VPlan analysis. + // TODO: Apply to all recipes in the future, to replace legacy uniformity + // analysis. + auto Users = collectUsersRecursively(PhiR); + for (VPUser *U : reverse(Users)) { + auto *Def = dyn_cast(U); + auto *RepR = dyn_cast(U); + // Skip recipes that shouldn't be narrowed. + if (!Def || !isa(Def) || + Def->getNumUsers() == 0 || !Def->getUnderlyingValue() || + (RepR && (RepR->isSingleScalar() || RepR->isPredicated()))) + continue; + + // Skip recipes that may have other lanes than their first used. + if (!vputils::isSingleScalar(Def) && !vputils::onlyFirstLaneUsed(Def)) + continue; + + auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(), + Def->operands(), /*IsUniform*/ true, + /*Mask*/ nullptr, /*Flags*/ *Def); + Clone->insertAfter(Def); + Def->replaceAllUsesWith(Clone); + } + // Replace wide pointer inductions which have only their scalars used by // PtrAdd(IndStart, ScalarIVSteps (0, Step)). if (auto *PtrIV = dyn_cast(&Phi)) { @@ -1522,11 +1547,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { continue; } - // Skip recipes that aren't single scalars and don't just have their first - // lane used. - if (!vputils::isSingleScalar(RepOrWidenR) && - (!vputils::onlyFirstLaneUsed(RepOrWidenR) || - RepOrWidenR->getNumUsers() == 0)) + // Skip recipes that aren't single scalars. + if (!vputils::isSingleScalar(RepOrWidenR)) continue; // Skip recipes for which conversion to single-scalar does introduce