diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c48ad3effcf88..ec75c02a3a092 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -33,6 +33,7 @@ #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist.h" #include "llvm/ADT/ilist_node.h" +#include "llvm/Analysis/IVDescriptors.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/DebugLoc.h" @@ -47,7 +48,6 @@ namespace llvm { class BasicBlock; class DominatorTree; -class InductionDescriptor; class InnerLoopVectorizer; class IRBuilderBase; class LoopInfo; @@ -1848,9 +1848,11 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe { return true; } - /// Check if the induction described by \p ID is canonical, i.e. has the same - /// start, step (of 1), and type as the canonical IV. - bool isCanonical(const InductionDescriptor &ID, Type *Ty) const; + /// Check if the induction described by \p Kind, /p Start and \p Step is + /// canonical, i.e. has the same start, step (of 1), and type as the + /// canonical IV. + bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, + VPValue *Step, Type *Ty) const; }; /// A recipe for generating the active lane mask for the vector loop that is diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index edc1e362d93d2..0ceb2fb8740e6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -735,8 +735,13 @@ void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, const Twine &Indent, #endif bool VPWidenIntOrFpInductionRecipe::isCanonical() const { + // The step may be defined by a recipe in the preheader (e.g. if it requires + // SCEV expansion), but for the canonical induction the step is required to be + // 1, which is represented as live-in. + if (getStepValue()->getDefiningRecipe()) + return false; + auto *StepC = dyn_cast(getStepValue()->getLiveInIRValue()); auto *StartC = dyn_cast(getStartValue()->getLiveInIRValue()); - auto *StepC = dyn_cast(getInductionDescriptor().getStep()); return StartC && StartC->isZero() && StepC && StepC->isOne(); } @@ -1091,20 +1096,22 @@ void VPCanonicalIVPHIRecipe::print(raw_ostream &O, const Twine &Indent, } #endif -bool VPCanonicalIVPHIRecipe::isCanonical(const InductionDescriptor &ID, - Type *Ty) const { - if (Ty != getScalarType()) +bool VPCanonicalIVPHIRecipe::isCanonical( + InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, + Type *Ty) const { + // The types must match and it must be an integer induction. + if (Ty != getScalarType() || Kind != InductionDescriptor::IK_IntInduction) + return false; + // Start must match the start value of this canonical induction. + if (Start != getStartValue()) return false; - // The start value of ID must match the start value of this canonical - // induction. - if (getStartValue()->getLiveInIRValue() != ID.getStartValue()) + + // If the step is defined by a recipe, it is not a ConstantInt. + if (Step->getDefiningRecipe()) return false; - ConstantInt *Step = ID.getConstIntStepValue(); - // ID must also be incremented by one. IK_IntInduction always increment the - // induction by Step, but the binary op may not be set. - return ID.getKind() == InductionDescriptor::IK_IntInduction && Step && - Step->isOne(); + ConstantInt *StepC = dyn_cast(Step->getLiveInIRValue()); + return StepC && StepC->isOne(); } bool VPWidenPointerInductionRecipe::onlyScalarsGenerated(ElementCount VF) { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index f091955174cf9..abd8b4f8cf2f2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -520,7 +520,8 @@ void VPlanTransforms::optimizeInductions(VPlan &Plan, ScalarEvolution &SE) { VPValue *Step = vputils::getOrCreateVPValueForSCEVExpr(Plan, ID.getStep(), SE); VPValue *BaseIV = CanonicalIV; - if (!CanonicalIV->isCanonical(ID, ResultTy)) { + if (!CanonicalIV->isCanonical(ID.getKind(), WideIV->getStartValue(), Step, + ResultTy)) { BaseIV = new VPDerivedIVRecipe(ID, WideIV->getStartValue(), CanonicalIV, Step, ResultTy); HeaderVPBB->insert(BaseIV->getDefiningRecipe(), IP);