diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 5dc3175382254..f533a47150a7b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -63,9 +63,11 @@ class VPBuilder { } VPInstruction *createInstruction(unsigned Opcode, - ArrayRef Operands, DebugLoc DL, + ArrayRef Operands, + const VPIRMetadata &MD, DebugLoc DL, const Twine &Name = "") { - return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name)); + return tryInsertInstruction( + new VPInstruction(Opcode, Operands, {}, MD, DL, Name)); } public: @@ -150,17 +152,17 @@ class VPBuilder { /// its underlying Instruction. VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, Instruction *Inst = nullptr, + const VPIRMetadata &MD = {}, + DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - DebugLoc DL = DebugLoc::getUnknown(); - if (Inst) - DL = Inst->getDebugLoc(); - VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name); + VPInstruction *NewVPInst = tryInsertInstruction( + new VPInstruction(Opcode, Operands, {}, MD, DL, Name)); NewVPInst->setUnderlyingValue(Inst); return NewVPInst; } VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") { - return createInstruction(Opcode, Operands, DL, Name); + return createInstruction(Opcode, Operands, {}, DL, Name); } VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, const VPIRFlags &Flags, @@ -174,8 +176,8 @@ class VPBuilder { Type *ResultTy, const VPIRFlags &Flags = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return tryInsertInstruction( - new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name)); + return tryInsertInstruction(new VPInstructionWithType( + Opcode, Operands, ResultTy, Flags, {}, DL, Name)); } VPInstruction *createOverflowingOp( @@ -189,13 +191,14 @@ class VPBuilder { VPInstruction *createNot(VPValue *Operand, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return createInstruction(VPInstruction::Not, {Operand}, DL, Name); + return createInstruction(VPInstruction::Not, {Operand}, {}, DL, Name); } VPInstruction *createAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name); + return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, {}, DL, + Name); } VPInstruction *createOr(VPValue *LHS, VPValue *RHS, @@ -210,20 +213,18 @@ class VPBuilder { VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return tryInsertInstruction( - new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name)); + return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name); } VPInstruction * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "", std::optional FMFs = std::nullopt) { - auto *Select = - FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - *FMFs, {}, DL, Name) - : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - DL, Name); - return tryInsertInstruction(Select); + if (!FMFs) + return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL, + Name); + return tryInsertInstruction(new VPInstruction( + Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name)); } /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A @@ -306,7 +307,7 @@ class VPBuilder { const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}) { return tryInsertInstruction( - new VPInstructionWithType(Opcode, Op, ResultTy, DL, Flags, Metadata)); + new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL)); } VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy, diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 10bd6cd471152..356d759b94799 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7616,14 +7616,13 @@ VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI, } if (VPI->getOpcode() == Instruction::Load) { auto *Load = cast(I); - return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, - VPIRMetadata(*Load, LVer), I->getDebugLoc()); + return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, *VPI, + VPI->getDebugLoc()); } StoreInst *Store = cast(I); return new VPWidenStoreRecipe(*Store, Ptr, VPI->getOperand(0), Mask, - Consecutive, Reverse, - VPIRMetadata(*Store, LVer), VPI->getDebugLoc()); + Consecutive, Reverse, *VPI, VPI->getDebugLoc()); } /// Creates a VPWidenIntOrFpInductionRecipe for \p PhiR. If needed, it will @@ -7751,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI, }, Range); if (ShouldUseVectorIntrinsic) - return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), + return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, VPI->getDebugLoc()); Function *Variant = nullptr; @@ -7843,7 +7842,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc()); Ops[1] = SafeRHS; - return new VPWidenRecipe(*I, Ops); + return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc()); } [[fallthrough]]; } @@ -7889,7 +7888,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { // For other binops, the legacy cost model only checks the second operand. NewOps[1] = GetConstantViaSCEV(NewOps[1]); } - return new VPWidenRecipe(*I, NewOps); + return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc()); } case Instruction::ExtractValue: { SmallVector NewOps(VPI->operands()); @@ -7897,7 +7896,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index"); unsigned Idx = EVI->getIndices()[0]; NewOps.push_back(Plan.getConstantInt(32, Idx)); - return new VPWidenRecipe(*I, NewOps); + return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc()); } }; } @@ -7981,8 +7980,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI, assert((Range.Start.isScalar() || !IsUniform || !IsPredicated || (Range.Start.isScalable() && isa(I))) && "Should not predicate a uniform recipe"); - auto *Recipe = new VPReplicateRecipe(I, VPI->operands(), IsUniform, - BlockInMask, VPIRMetadata(*I, LVer)); + auto *Recipe = + new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI); return Recipe; } @@ -8235,13 +8234,14 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, return new VPWidenGEPRecipe(cast(Instr), R->operands()); if (VPI->getOpcode() == Instruction::Select) - return new VPWidenSelectRecipe(*cast(Instr), R->operands()); + return new VPWidenSelectRecipe(*cast(Instr), R->operands(), + *VPI); if (Instruction::isCast(VPI->getOpcode())) { auto *CastR = cast(R); auto *CI = cast(Instr); return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0), - CastR->getResultType(), *CI); + CastR->getResultType(), *CI, *VPI); } return tryToWiden(VPI); @@ -8269,7 +8269,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction, SmallVector Ops; Ops.push_back(Plan.getOrAddLiveIn(Zero)); Ops.push_back(BinOp); - BinOp = new VPWidenRecipe(*ReductionI, Ops); + BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(), + ReductionI->getDebugLoc()); Builder.insert(BinOp->getDefiningRecipe()); ReductionOpcode = Instruction::Add; } @@ -8302,7 +8303,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, // candidates built later for specific VF ranges. auto VPlan0 = VPlanTransforms::buildVPlan0( OrigLoop, *LI, Legal->getWidestInductionType(), - getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE); + getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer); auto MaxVFTimes2 = MaxVF * 2; for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) { @@ -8408,7 +8409,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( // VPInstructions in the loop. // --------------------------------------------------------------------------- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder, BlockMaskCache, LVer); + Builder, BlockMaskCache); // TODO: Handle partial reductions with EVL tail folding. if (!CM.foldTailWithEVL()) RecipeBuilder.collectScaledReductions(Range); @@ -8453,9 +8454,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) { // Only create recipe for the final invariant store of the reduction. if (Legal->isInvariantStoreOfReduction(SI)) { - auto *Recipe = - new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */, - nullptr /*Mask*/, VPIRMetadata(*SI, LVer)); + auto *Recipe = new VPReplicateRecipe( + SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, + *cast(SingleDef)); Recipe->insertBefore(*MiddleVPBB, MBIP); } R.eraseFromParent(); @@ -8606,7 +8607,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) { // addScalarResumePhis. DenseMap BlockMaskCache; VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder, BlockMaskCache, nullptr /*LVer*/); + Builder, BlockMaskCache); for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { if (isa(&R)) continue; diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index a7000aff06379..87280b83fc0e5 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -84,10 +84,6 @@ class VPRecipeBuilder { /// A mapping of partial reduction exit instructions to their scaling factor. DenseMap ScaledReductionMap; - /// Loop versioning instance for getting noalias metadata guaranteed by - /// runtime checks. - LoopVersioning *LVer; - /// Check if \p I can be widened at the start of \p Range and possibly /// decrease the range such that the returned value holds for the entire \p /// Range. The function should not be called for memory instructions or calls. @@ -144,11 +140,9 @@ class VPRecipeBuilder { LoopVectorizationLegality *Legal, LoopVectorizationCostModel &CM, PredicatedScalarEvolution &PSE, VPBuilder &Builder, - DenseMap &BlockMaskCache, - LoopVersioning *LVer) + DenseMap &BlockMaskCache) : Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal), - CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache), - LVer(LVer) {} + CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache) {} std::optional getScalingForReduction(const Instruction *ExitInst) { auto It = ScaledReductionMap.find(ExitInst); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 67fa294d095bd..c81834e401726 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -65,7 +65,6 @@ class VPReplicateRecipe; class VPlanSlp; class Value; class LoopVectorizationCostModel; -class LoopVersioning; struct VPCostContext; @@ -958,10 +957,6 @@ class VPIRMetadata { /// \p I. VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); } - /// Adds metatadata that can be preserved from the original instruction - /// \p I and noalias metadata guaranteed by runtime checks using \p LVer. - VPIRMetadata(Instruction &I, LoopVersioning *LVer); - /// Copy constructor for cloning. VPIRMetadata(const VPIRMetadata &Other) = default; @@ -1120,11 +1115,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, public: VPInstruction(unsigned Opcode, ArrayRef Operands, - DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") - : VPInstruction(Opcode, Operands, {}, {}, DL, Name) {} - - VPInstruction(unsigned Opcode, ArrayRef Operands, - const VPIRFlags &Flags, const VPIRMetadata &MD = {}, + const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = ""); VP_CLASSOF_IMPL(VPDef::VPInstructionSC) @@ -1214,14 +1205,10 @@ class VPInstructionWithType : public VPInstruction { public: VPInstructionWithType(unsigned Opcode, ArrayRef Operands, - Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL, + Type *ResultTy, const VPIRFlags &Flags = {}, + const VPIRMetadata &Metadata = {}, + DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") - : VPInstruction(Opcode, Operands, Flags, {}, DL, Name), - ResultTy(ResultTy) {} - - VPInstructionWithType(unsigned Opcode, ArrayRef Operands, - Type *ResultTy, DebugLoc DL, const VPIRFlags &Flags, - const VPIRMetadata &Metadata, const Twine &Name = "") : VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name), ResultTy(ResultTy) {} @@ -1250,7 +1237,7 @@ class VPInstructionWithType : public VPInstruction { VPInstruction *clone() override { auto *New = new VPInstructionWithType(getOpcode(), operands(), getResultType(), - *this, getDebugLoc(), getName()); + *this, *this, getDebugLoc(), getName()); New->setUnderlyingValue(getUnderlyingValue()); return New; } @@ -1334,7 +1321,7 @@ class VPPhiAccessors { struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors { VPPhi(ArrayRef Operands, DebugLoc DL, const Twine &Name = "") - : VPInstruction(Instruction::PHI, Operands, DL, Name) {} + : VPInstruction(Instruction::PHI, Operands, {}, {}, DL, Name) {} static inline bool classof(const VPUser *U) { auto *VPI = dyn_cast(U); @@ -1478,9 +1465,10 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags, : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL), VPIRMetadata(Metadata), Opcode(Opcode) {} - VPWidenRecipe(Instruction &I, ArrayRef Operands) - : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I), - Opcode(I.getOpcode()) {} + VPWidenRecipe(Instruction &I, ArrayRef Operands, + const VPIRMetadata &Metadata, DebugLoc DL) + : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), + VPIRMetadata(Metadata), Opcode(I.getOpcode()) {} ~VPWidenRecipe() override = default; @@ -1521,13 +1509,12 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - CastInst &UI) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), VPIRMetadata(UI), - Opcode(Opcode), ResultTy(ResultTy) { + CastInst &UI, const VPIRMetadata &Metadata) + : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), + VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) { assert(UI.getOpcode() == Opcode && "opcode of underlying cast doesn't match"); } - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}, @@ -1590,18 +1577,23 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, + const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown()) : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), - VPIRMetadata(CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), + VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), MayReadFromMemory(CI.mayReadFromMemory()), MayWriteToMemory(CI.mayWriteToMemory()), MayHaveSideEffects(CI.mayHaveSideEffects()) {} VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, + const VPIRFlags &Flags = {}, + const VPIRMetadata &Metadata = {}, DebugLoc DL = DebugLoc::getUnknown()) - : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL), - VPIRMetadata(), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) { + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags, + DL), + VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID), + ResultTy(Ty) { LLVMContext &Ctx = Ty->getContext(); AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID); MemoryEffects ME = Attrs.getMemoryEffects(); @@ -1617,9 +1609,10 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { VPWidenIntrinsicRecipe *clone() override { if (Value *CI = getUnderlyingValue()) return new VPWidenIntrinsicRecipe(*cast(CI), VectorIntrinsicID, - operands(), ResultTy, getDebugLoc()); + operands(), ResultTy, *this, + getDebugLoc()); return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy, - getDebugLoc()); + *this, *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC) @@ -1760,15 +1753,16 @@ class VPHistogramRecipe : public VPRecipeBase { /// instruction. struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { - VPWidenSelectRecipe(SelectInst &I, ArrayRef Operands) + VPWidenSelectRecipe(SelectInst &I, ArrayRef Operands, + const VPIRMetadata &MD = {}) : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I), - VPIRMetadata(I) {} + VPIRMetadata(MD) {} ~VPWidenSelectRecipe() override = default; VPWidenSelectRecipe *clone() override { return new VPWidenSelectRecipe(*cast(getUnderlyingInstr()), - operands()); + operands(), *this); } VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index aed85271350c8..612202d049774 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -22,6 +22,7 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/Transforms/Utils/LoopVersioning.h" #define DEBUG_TYPE "vplan" @@ -37,6 +38,9 @@ class PlainCFGBuilder { // Loop Info analysis. LoopInfo *LI; + // Loop versioning for alias metadata. + LoopVersioning *LVer; + // Vectorization plan that we are working on. std::unique_ptr Plan; @@ -65,8 +69,8 @@ class PlainCFGBuilder { void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB); public: - PlainCFGBuilder(Loop *Lp, LoopInfo *LI) - : TheLoop(Lp), LI(LI), Plan(std::make_unique(Lp)) {} + PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer) + : TheLoop(Lp), LI(LI), LVer(LVer), Plan(std::make_unique(Lp)) {} /// Build plain CFG for TheLoop and connect it to Plan's entry. std::unique_ptr buildPlainCFG(); @@ -186,7 +190,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // recipes. if (Br->isConditional()) { VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); - VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst); + VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, + VPIRMetadata(*Inst), Inst->getDebugLoc()); } // Skip the rest of the Instruction processing for Branch instructions. @@ -200,7 +205,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, SmallVector Ops = {getOrCreateVPOperand(SI->getCondition())}; for (auto Case : SI->cases()) Ops.push_back(getOrCreateVPOperand(Case.getCaseValue())); - VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst); + VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, + VPIRMetadata(*Inst), Inst->getDebugLoc()); continue; } @@ -228,6 +234,18 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock())); } } else { + // Build VPIRMetadata from the instruction and add loop versioning + // metadata for loads and stores. + VPIRMetadata MD(*Inst); + if (isa(Inst) && LVer) { + const auto &[AliasScopeMD, NoAliasMD] = + LVer->getNoAliasMetadataFor(Inst); + if (AliasScopeMD) + MD.setMetadata(LLVMContext::MD_alias_scope, AliasScopeMD); + if (NoAliasMD) + MD.setMetadata(LLVMContext::MD_noalias, NoAliasMD); + } + // Translate LLVM-IR operands into VPValue operands and set them in the // new VPInstruction. SmallVector VPOperands; @@ -236,12 +254,14 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, if (auto *CI = dyn_cast(Inst)) { NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0], - CI->getType(), CI->getDebugLoc()); + CI->getType(), CI->getDebugLoc(), + {}, MD); NewR->setUnderlyingValue(CI); } else { // Build VPInstruction for any arbitrary Instruction without specific // representation in VPlan. - NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst); + NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD, + Inst->getDebugLoc()); } } @@ -537,8 +557,9 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL, std::unique_ptr VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, - DebugLoc IVDL, PredicatedScalarEvolution &PSE) { - PlainCFGBuilder Builder(TheLoop, &LI); + DebugLoc IVDL, PredicatedScalarEvolution &PSE, + LoopVersioning *LVer) { + PlainCFGBuilder Builder(TheLoop, &LI, LVer); std::unique_ptr VPlan0 = Builder.buildPlainCFG(); addInitialSkeleton(*VPlan0, InductionTy, IVDL, PSE, TheLoop); return VPlan0; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index e2a8e495d5ed5..fca6554ad77c6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -36,7 +36,6 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/LoopUtils.h" -#include "llvm/Transforms/Utils/LoopVersioning.h" #include using namespace llvm; @@ -1674,17 +1673,6 @@ void VPIRPhi::printRecipe(raw_ostream &O, const Twine &Indent, } #endif -VPIRMetadata::VPIRMetadata(Instruction &I, LoopVersioning *LVer) - : VPIRMetadata(I) { - if (!LVer || !isa(&I)) - return; - const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(&I); - if (AliasScopeMD) - Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD); - if (NoAliasMD) - Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD); -} - void VPIRMetadata::applyMetadata(Instruction &I) const { for (const auto &[Kind, Node] : Metadata) I.setMetadata(Kind, Node); diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp index 1453c6623625b..3b5cc9fcb9820 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp @@ -517,7 +517,8 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef Values) { assert(CombinedOperands.size() > 0 && "Need more some operands"); auto *Inst = cast(Values[0])->getUnderlyingInstr(); - auto *VPI = new VPInstruction(Opcode, CombinedOperands, Inst->getDebugLoc()); + auto *VPI = + new VPInstruction(Opcode, CombinedOperands, {}, {}, Inst->getDebugLoc()); LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " " << Values[0] << "\n"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 3e2c47e4556a6..89118b49bed44 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -85,20 +85,19 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes( Ingredient.getDebugLoc()); } } else { - assert(isa(&Ingredient) && - "only VPInstructions expected here"); + auto *VPI = cast(&Ingredient); assert(!isa(Inst) && "phis should be handled above"); // Create VPWidenMemoryRecipe for loads and stores. if (LoadInst *Load = dyn_cast(Inst)) { NewRecipe = new VPWidenLoadRecipe( *Load, Ingredient.getOperand(0), nullptr /*Mask*/, - false /*Consecutive*/, false /*Reverse*/, VPIRMetadata(*Load), + false /*Consecutive*/, false /*Reverse*/, *VPI, Ingredient.getDebugLoc()); } else if (StoreInst *Store = dyn_cast(Inst)) { NewRecipe = new VPWidenStoreRecipe( *Store, Ingredient.getOperand(1), Ingredient.getOperand(0), - nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, - VPIRMetadata(*Store), Ingredient.getDebugLoc()); + nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, *VPI, + Ingredient.getDebugLoc()); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); } else if (CallInst *CI = dyn_cast(Inst)) { @@ -107,15 +106,17 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes( return false; NewRecipe = new VPWidenIntrinsicRecipe( *CI, getVectorIntrinsicIDForCall(CI, &TLI), - drop_end(Ingredient.operands()), CI->getType(), + drop_end(Ingredient.operands()), CI->getType(), *VPI, CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { - NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); + NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands(), *VPI); } else if (auto *CI = dyn_cast(Inst)) { - NewRecipe = new VPWidenCastRecipe( - CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI); + NewRecipe = + new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0), + CI->getType(), *CI, *VPI); } else { - NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands()); + NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI, + Ingredient.getDebugLoc()); } } @@ -1705,8 +1706,9 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF, Ops.append({ALM, Plan.getOrAddLiveIn( ConstantInt::get(IntegerType::getInt64Ty(Ctx), VF.getKnownMinValue() * Part))}); - auto *Ext = new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops, - IntegerType::getInt1Ty(Ctx), DL); + auto *Ext = + new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops, + IntegerType::getInt1Ty(Ctx), {}, {}, DL); Extracts[Part] = Ext; Ext->insertAfter(ALM); } @@ -1845,7 +1847,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF, // The vector region contains header phis for which we cannot remove the // loop region yet. auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()}, - Term->getDebugLoc()); + {}, {}, Term->getDebugLoc()); ExitingVPBB->appendRecipe(BOC); } @@ -2679,13 +2681,13 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, m_Select(m_Specific(HeaderMask), m_VPValue(LHS), m_VPValue(RHS)))) return new VPWidenIntrinsicRecipe( Intrinsic::vp_merge, {Plan->getTrue(), LHS, RHS, &EVL}, - TypeInfo.inferScalarType(LHS), CurRecipe.getDebugLoc()); + TypeInfo.inferScalarType(LHS), {}, {}, CurRecipe.getDebugLoc()); if (match(&CurRecipe, m_Select(m_RemoveMask(HeaderMask, Mask), m_VPValue(LHS), m_VPValue(RHS)))) return new VPWidenIntrinsicRecipe( Intrinsic::vp_merge, {Mask, LHS, RHS, &EVL}, - TypeInfo.inferScalarType(LHS), CurRecipe.getDebugLoc()); + TypeInfo.inferScalarType(LHS), {}, {}, CurRecipe.getDebugLoc()); return nullptr; } @@ -2753,7 +2755,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe( Intrinsic::experimental_vp_splice, {V1, V2, Imm, Plan.getTrue(), PrevEVL, &EVL}, - TypeInfo.inferScalarType(R.getVPSingleValue()), R.getDebugLoc()); + TypeInfo.inferScalarType(R.getVPSingleValue()), {}, {}, + R.getDebugLoc()); VPSplice->insertBefore(&R); R.getVPSingleValue()->replaceAllUsesWith(VPSplice); ToErase.push_back(&R); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index e3bde8a47dcbc..a44a4f69c917b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -23,6 +23,7 @@ namespace llvm { class InductionDescriptor; class Instruction; +class LoopVersioning; class PHINode; class ScalarEvolution; class PredicatedScalarEvolution; @@ -99,7 +100,7 @@ struct VPlanTransforms { /// >[ ] <-- original loop exit block(s), wrapped in VPIRBasicBlocks. LLVM_ABI_FOR_TEST static std::unique_ptr buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL, - PredicatedScalarEvolution &PSE); + PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr); /// Update \p Plan to account for all early exits. LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan, diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index ee7fa175ca918..0e76c64f09f59 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1009,7 +1009,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) { SmallVector Args; Args.push_back(Op1); Args.push_back(Op2); - VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end())); + VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc()); checkVPRecipeCastImpl(&WidenR); delete AI; @@ -1092,7 +1092,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) { IntegerType *Int64 = IntegerType::get(C, 64); auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64); VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1)); - VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast); + VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast, {}); checkVPRecipeCastImpl(&Recipe); delete Cast; @@ -1263,7 +1263,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { SmallVector Args; Args.push_back(Op1); Args.push_back(Op2); - VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end())); + VPWidenRecipe Recipe(*AI, Args, VPIRMetadata(), DebugLoc()); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1468,7 +1468,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) { Args.push_back(ExtVPV1); Args.push_back(ExtVPV2); VPWidenRecipe *WidenR = - new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end())); + new VPWidenRecipe(*AI, Args, VPIRMetadata(), DebugLoc()); VPBB1->appendRecipe(WidenR); {