diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index acdb37996a443..2b3f4af8fa3ad 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -189,19 +189,7 @@ static bool sinkScalarOperands(VPlan &Plan) { if (NeedsDuplicating) { if (ScalarVFOnly) continue; - VPSingleDefRecipe *Clone; - if (auto *SinkCandidateRepR = - dyn_cast(SinkCandidate)) { - // TODO: Handle converting to uniform recipes as separate transform, - // then cloning should be sufficient here. - Instruction *I = SinkCandidate->getUnderlyingInstr(); - Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true, - nullptr /*Mask*/, *SinkCandidateRepR); - // TODO: add ".cloned" suffix to name of Clone's VPValue. - } else { - Clone = SinkCandidate->clone(); - } - + VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(SinkCandidate); Clone->insertBefore(SinkCandidate); SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { return cast(&U)->getParent() != SinkTo; @@ -666,8 +654,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { if (!vputils::isSingleScalar(Def) && !vputils::onlyFirstLaneUsed(Def)) continue; - auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(), - Def->operands(), /*IsUniform*/ true); + VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(Def); Clone->insertAfter(Def); Def->replaceAllUsesWith(Clone); } @@ -1309,15 +1296,14 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { auto *RepOrWidenR = cast(&R); if (RepR && isa(RepR->getUnderlyingInstr()) && vputils::isSingleScalar(RepR->getOperand(1))) { - auto *Clone = new VPReplicateRecipe( - RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(), - true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/); + auto *Clone = + cast(vputils::getSingleScalarClone(RepOrWidenR)); Clone->insertBefore(RepOrWidenR); auto *Ext = new VPInstruction(VPInstruction::ExtractLastElement, {Clone->getOperand(0)}); Ext->insertBefore(Clone); Clone->setOperand(0, Ext); - RepR->eraseFromParent(); + RepOrWidenR->eraseFromParent(); continue; } @@ -1332,9 +1318,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { })) continue; - auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(), - RepOrWidenR->operands(), - true /*IsSingleScalar*/); + VPSingleDefRecipe *Clone = vputils::getSingleScalarClone(RepOrWidenR); Clone->insertBefore(RepOrWidenR); RepOrWidenR->replaceAllUsesWith(Clone); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp index 059993043dcda..45addf76f9eae 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.cpp @@ -250,3 +250,24 @@ vputils::getRecipesForUncountableExit(VPlan &Plan, return UncountableCondition; } + +VPSingleDefRecipe *vputils::getSingleScalarClone(VPSingleDefRecipe *R) { + return TypeSwitch(R) + .Case([](auto *I) { + return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(), + /*IsSingleScalar*/ true, + /*Mask*/ nullptr, + /*Metadata*/ *I); + }) + .Case([](auto *I) { + // WidenGEP does not have metadata. + return new VPReplicateRecipe(I->getUnderlyingInstr(), I->operands(), + /*IsSingleScalar*/ true, /*Mask*/ nullptr); + }) + .Case([](auto *I) { return I->clone(); }) + .Default([](auto *I) { + llvm_unreachable("Recipe not convertible to single-scalar"); + return nullptr; + }); +} diff --git a/llvm/lib/Transforms/Vectorize/VPlanUtils.h b/llvm/lib/Transforms/Vectorize/VPlanUtils.h index 0222b0aa81063..0593c5355a6ec 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUtils.h +++ b/llvm/lib/Transforms/Vectorize/VPlanUtils.h @@ -116,6 +116,10 @@ std::optional getRecipesForUncountableExit(VPlan &Plan, SmallVectorImpl &Recipes, SmallVectorImpl &GEPs); + +/// Returns a single-scalar version of \p R, creating a fresh single-scalar +/// VPReplicateRecipe or just cloning the recipe. +VPSingleDefRecipe *getSingleScalarClone(VPSingleDefRecipe *R); } // namespace vputils //===----------------------------------------------------------------------===//