From 033f4308374d0eab9d046da40536d074f1a4ca9d Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Wed, 5 Nov 2025 21:35:12 +0000 Subject: [PATCH 1/5] [VPlan] Populate and use VPIRMetadata from VPInstructions (NFC) Update VPlan to populate VPIRMetadata during VPInstruction construction and use it when creating widened recipes, instead of constructing VPIRMetadata from the underlying IR instruction each time. This centralizes VPIRMetadata in VPInstructions and ensures metadata is consistently available throughout VPlan transformations. --- .../Vectorize/LoopVectorizationPlanner.h | 17 +++---- .../Transforms/Vectorize/LoopVectorize.cpp | 36 +++++++-------- .../Transforms/Vectorize/VPRecipeBuilder.h | 10 +--- llvm/lib/Transforms/Vectorize/VPlan.h | 46 +++++++++---------- .../Vectorize/VPlanConstruction.cpp | 35 ++++++++++---- llvm/lib/Transforms/Vectorize/VPlanSLP.cpp | 3 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 29 ++++++------ .../Transforms/Vectorize/VPlanTransforms.h | 3 +- .../Transforms/Vectorize/VPlanTest.cpp | 8 ++-- 9 files changed, 100 insertions(+), 87 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 5dc3175382254..1843338d69ff4 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -65,7 +65,8 @@ class VPBuilder { VPInstruction *createInstruction(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") { - return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name)); + return tryInsertInstruction( + new VPInstruction(Opcode, Operands, {}, DL, Name)); } public: @@ -150,11 +151,11 @@ class VPBuilder { /// its underlying Instruction. VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, Instruction *Inst = nullptr, + const VPIRMetadata &MD = {}, + DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - DebugLoc DL = DebugLoc::getUnknown(); - if (Inst) - DL = Inst->getDebugLoc(); - VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name); + VPInstruction *NewVPInst = + tryInsertInstruction(new VPInstruction(Opcode, Operands, MD, DL, Name)); NewVPInst->setUnderlyingValue(Inst); return NewVPInst; } @@ -211,7 +212,7 @@ class VPBuilder { DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { return tryInsertInstruction( - new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name)); + new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, {}, DL, Name)); } VPInstruction * @@ -222,7 +223,7 @@ class VPBuilder { FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name) : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - DL, Name); + {}, DL, Name); return tryInsertInstruction(Select); } @@ -328,7 +329,7 @@ class VPBuilder { else if (Opcode == Instruction::ZExt) Flags = VPIRFlags::NonNegFlagsTy(false); return tryInsertInstruction( - new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags)); + new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags)); } VPScalarIVStepsRecipe * diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 10bd6cd471152..fadc0918f264d 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7616,14 +7616,13 @@ VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI, } if (VPI->getOpcode() == Instruction::Load) { auto *Load = cast(I); - return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, - VPIRMetadata(*Load, LVer), I->getDebugLoc()); + return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, *VPI, + VPI->getDebugLoc()); } StoreInst *Store = cast(I); return new VPWidenStoreRecipe(*Store, Ptr, VPI->getOperand(0), Mask, - Consecutive, Reverse, - VPIRMetadata(*Store, LVer), VPI->getDebugLoc()); + Consecutive, Reverse, *VPI, VPI->getDebugLoc()); } /// Creates a VPWidenIntOrFpInductionRecipe for \p PhiR. If needed, it will @@ -7751,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI, }, Range); if (ShouldUseVectorIntrinsic) - return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), + return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, VPI->getDebugLoc()); Function *Variant = nullptr; @@ -7843,7 +7842,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc()); Ops[1] = SafeRHS; - return new VPWidenRecipe(*I, Ops); + return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc()); } [[fallthrough]]; } @@ -7889,7 +7888,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { // For other binops, the legacy cost model only checks the second operand. NewOps[1] = GetConstantViaSCEV(NewOps[1]); } - return new VPWidenRecipe(*I, NewOps); + return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc()); } case Instruction::ExtractValue: { SmallVector NewOps(VPI->operands()); @@ -7897,7 +7896,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index"); unsigned Idx = EVI->getIndices()[0]; NewOps.push_back(Plan.getConstantInt(32, Idx)); - return new VPWidenRecipe(*I, NewOps); + return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc()); } }; } @@ -7981,8 +7980,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI, assert((Range.Start.isScalar() || !IsUniform || !IsPredicated || (Range.Start.isScalable() && isa(I))) && "Should not predicate a uniform recipe"); - auto *Recipe = new VPReplicateRecipe(I, VPI->operands(), IsUniform, - BlockInMask, VPIRMetadata(*I, LVer)); + auto *Recipe = + new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI); return Recipe; } @@ -8241,7 +8240,7 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, auto *CastR = cast(R); auto *CI = cast(Instr); return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0), - CastR->getResultType(), *CI); + CastR->getResultType(), CI, *VPI, *VPI); } return tryToWiden(VPI); @@ -8269,7 +8268,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction, SmallVector Ops; Ops.push_back(Plan.getOrAddLiveIn(Zero)); Ops.push_back(BinOp); - BinOp = new VPWidenRecipe(*ReductionI, Ops); + BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(), + ReductionI->getDebugLoc()); Builder.insert(BinOp->getDefiningRecipe()); ReductionOpcode = Instruction::Add; } @@ -8302,7 +8302,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF, // candidates built later for specific VF ranges. auto VPlan0 = VPlanTransforms::buildVPlan0( OrigLoop, *LI, Legal->getWidestInductionType(), - getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE); + getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer); auto MaxVFTimes2 = MaxVF * 2; for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) { @@ -8408,7 +8408,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( // VPInstructions in the loop. // --------------------------------------------------------------------------- VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder, BlockMaskCache, LVer); + Builder, BlockMaskCache); // TODO: Handle partial reductions with EVL tail folding. if (!CM.foldTailWithEVL()) RecipeBuilder.collectScaledReductions(Range); @@ -8453,9 +8453,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) { // Only create recipe for the final invariant store of the reduction. if (Legal->isInvariantStoreOfReduction(SI)) { - auto *Recipe = - new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */, - nullptr /*Mask*/, VPIRMetadata(*SI, LVer)); + auto *Recipe = new VPReplicateRecipe( + SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, + *cast(SingleDef)); Recipe->insertBefore(*MiddleVPBB, MBIP); } R.eraseFromParent(); @@ -8606,7 +8606,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) { // addScalarResumePhis. DenseMap BlockMaskCache; VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE, - Builder, BlockMaskCache, nullptr /*LVer*/); + Builder, BlockMaskCache); for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) { if (isa(&R)) continue; diff --git a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h index a7000aff06379..87280b83fc0e5 100644 --- a/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h +++ b/llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h @@ -84,10 +84,6 @@ class VPRecipeBuilder { /// A mapping of partial reduction exit instructions to their scaling factor. DenseMap ScaledReductionMap; - /// Loop versioning instance for getting noalias metadata guaranteed by - /// runtime checks. - LoopVersioning *LVer; - /// Check if \p I can be widened at the start of \p Range and possibly /// decrease the range such that the returned value holds for the entire \p /// Range. The function should not be called for memory instructions or calls. @@ -144,11 +140,9 @@ class VPRecipeBuilder { LoopVectorizationLegality *Legal, LoopVectorizationCostModel &CM, PredicatedScalarEvolution &PSE, VPBuilder &Builder, - DenseMap &BlockMaskCache, - LoopVersioning *LVer) + DenseMap &BlockMaskCache) : Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal), - CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache), - LVer(LVer) {} + CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache) {} std::optional getScalingForReduction(const Instruction *ExitInst) { auto It = ScaledReductionMap.find(ExitInst); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 67fa294d095bd..fea9c9b7aa5e3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1119,10 +1119,6 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, #endif public: - VPInstruction(unsigned Opcode, ArrayRef Operands, - DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") - : VPInstruction(Opcode, Operands, {}, {}, DL, Name) {} - VPInstruction(unsigned Opcode, ArrayRef Operands, const VPIRFlags &Flags, const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = ""); @@ -1334,7 +1330,7 @@ class VPPhiAccessors { struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors { VPPhi(ArrayRef Operands, DebugLoc DL, const Twine &Name = "") - : VPInstruction(Instruction::PHI, Operands, DL, Name) {} + : VPInstruction(Instruction::PHI, Operands, {}, DL, Name) {} static inline bool classof(const VPUser *U) { auto *VPI = dyn_cast(U); @@ -1478,9 +1474,12 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags, : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL), VPIRMetadata(Metadata), Opcode(Opcode) {} - VPWidenRecipe(Instruction &I, ArrayRef Operands) - : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), VPIRMetadata(I), - Opcode(I.getOpcode()) {} + VPWidenRecipe(Instruction &I, ArrayRef Operands, + const VPIRMetadata &Metadata, DebugLoc DL) + : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, VPIRFlags(I), DL), + VPIRMetadata(Metadata), Opcode(I.getOpcode()) { + setUnderlyingValue(&I); + } ~VPWidenRecipe() override = default; @@ -1521,31 +1520,26 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - CastInst &UI) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), VPIRMetadata(UI), - Opcode(Opcode), ResultTy(ResultTy) { - assert(UI.getOpcode() == Opcode && - "opcode of underlying cast doesn't match"); - } - - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - const VPIRFlags &Flags = {}, + CastInst *UI = nullptr, const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}, DebugLoc DL = DebugLoc::getUnknown()) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL), + : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, + UI ? VPIRFlags(*UI) : Flags, + UI ? UI->getDebugLoc() : DL), VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) { assert(flagsValidForOpcode(Opcode) && "Set flags not supported for the provided opcode"); + assert((!UI || UI->getOpcode() == Opcode) && + "opcode of underlying cast doesn't match"); + setUnderlyingValue(UI); } ~VPWidenCastRecipe() override = default; VPWidenCastRecipe *clone() override { - auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this, - *this, getDebugLoc()); - if (auto *UV = getUnderlyingValue()) - New->setUnderlyingValue(UV); - return New; + return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, + cast_or_null(getUnderlyingValue()), + *this, *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCastSC) @@ -1590,9 +1584,10 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, + const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown()) : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), - VPIRMetadata(CI), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), + VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), MayReadFromMemory(CI.mayReadFromMemory()), MayWriteToMemory(CI.mayWriteToMemory()), MayHaveSideEffects(CI.mayHaveSideEffects()) {} @@ -1617,7 +1612,8 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { VPWidenIntrinsicRecipe *clone() override { if (Value *CI = getUnderlyingValue()) return new VPWidenIntrinsicRecipe(*cast(CI), VectorIntrinsicID, - operands(), ResultTy, getDebugLoc()); + operands(), ResultTy, *this, + getDebugLoc()); return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy, getDebugLoc()); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index aed85271350c8..a3179aacaa6ba 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -22,6 +22,7 @@ #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/MDBuilder.h" +#include "llvm/Transforms/Utils/LoopVersioning.h" #define DEBUG_TYPE "vplan" @@ -37,6 +38,9 @@ class PlainCFGBuilder { // Loop Info analysis. LoopInfo *LI; + // Loop versioning for alias metadata. + LoopVersioning *LVer; + // Vectorization plan that we are working on. std::unique_ptr Plan; @@ -65,8 +69,8 @@ class PlainCFGBuilder { void createVPInstructionsForVPBB(VPBasicBlock *VPBB, BasicBlock *BB); public: - PlainCFGBuilder(Loop *Lp, LoopInfo *LI) - : TheLoop(Lp), LI(LI), Plan(std::make_unique(Lp)) {} + PlainCFGBuilder(Loop *Lp, LoopInfo *LI, LoopVersioning *LVer) + : TheLoop(Lp), LI(LI), LVer(LVer), Plan(std::make_unique(Lp)) {} /// Build plain CFG for TheLoop and connect it to Plan's entry. std::unique_ptr buildPlainCFG(); @@ -186,7 +190,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // recipes. if (Br->isConditional()) { VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); - VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst); + VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, + VPIRMetadata(*Inst), Inst->getDebugLoc()); } // Skip the rest of the Instruction processing for Branch instructions. @@ -200,7 +205,8 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, SmallVector Ops = {getOrCreateVPOperand(SI->getCondition())}; for (auto Case : SI->cases()) Ops.push_back(getOrCreateVPOperand(Case.getCaseValue())); - VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst); + VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, + VPIRMetadata(*Inst), Inst->getDebugLoc()); continue; } @@ -228,6 +234,18 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, VPPredToIncomingValue.lookup(Pred->getExitingBasicBlock())); } } else { + // Build VPIRMetadata from the instruction and add loop versioning + // metadata for loads and stores. + VPIRMetadata MD(*Inst); + if (isa(Inst) && LVer) { + const auto &[AliasScopeMD, NoAliasMD] = + LVer->getNoAliasMetadataFor(Inst); + if (AliasScopeMD) + MD.addMetadata(LLVMContext::MD_alias_scope, AliasScopeMD); + if (NoAliasMD) + MD.addMetadata(LLVMContext::MD_noalias, NoAliasMD); + } + // Translate LLVM-IR operands into VPValue operands and set them in the // new VPInstruction. SmallVector VPOperands; @@ -236,12 +254,12 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, if (auto *CI = dyn_cast(Inst)) { NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0], - CI->getType(), CI->getDebugLoc()); + CI->getType(), CI->getDebugLoc(), {}, MD); NewR->setUnderlyingValue(CI); } else { // Build VPInstruction for any arbitrary Instruction without specific // representation in VPlan. - NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst); + NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD, Inst->getDebugLoc()); } } @@ -537,8 +555,9 @@ static void addInitialSkeleton(VPlan &Plan, Type *InductionTy, DebugLoc IVDL, std::unique_ptr VPlanTransforms::buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, - DebugLoc IVDL, PredicatedScalarEvolution &PSE) { - PlainCFGBuilder Builder(TheLoop, &LI); + DebugLoc IVDL, PredicatedScalarEvolution &PSE, + LoopVersioning *LVer) { + PlainCFGBuilder Builder(TheLoop, &LI, LVer); std::unique_ptr VPlan0 = Builder.buildPlainCFG(); addInitialSkeleton(*VPlan0, InductionTy, IVDL, PSE, TheLoop); return VPlan0; diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp index 1453c6623625b..3b8ff2515a3f7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp @@ -517,7 +517,8 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef Values) { assert(CombinedOperands.size() > 0 && "Need more some operands"); auto *Inst = cast(Values[0])->getUnderlyingInstr(); - auto *VPI = new VPInstruction(Opcode, CombinedOperands, Inst->getDebugLoc()); + auto *VPI = + new VPInstruction(Opcode, CombinedOperands, {}, Inst->getDebugLoc()); LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " " << Values[0] << "\n"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 3e2c47e4556a6..41481b207e8e2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -85,20 +85,19 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes( Ingredient.getDebugLoc()); } } else { - assert(isa(&Ingredient) && - "only VPInstructions expected here"); + auto *VPI = cast(&Ingredient); assert(!isa(Inst) && "phis should be handled above"); // Create VPWidenMemoryRecipe for loads and stores. if (LoadInst *Load = dyn_cast(Inst)) { NewRecipe = new VPWidenLoadRecipe( *Load, Ingredient.getOperand(0), nullptr /*Mask*/, - false /*Consecutive*/, false /*Reverse*/, VPIRMetadata(*Load), + false /*Consecutive*/, false /*Reverse*/, *VPI, Ingredient.getDebugLoc()); } else if (StoreInst *Store = dyn_cast(Inst)) { NewRecipe = new VPWidenStoreRecipe( *Store, Ingredient.getOperand(1), Ingredient.getOperand(0), - nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, - VPIRMetadata(*Store), Ingredient.getDebugLoc()); + nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, *VPI, + Ingredient.getDebugLoc()); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); } else if (CallInst *CI = dyn_cast(Inst)) { @@ -107,15 +106,17 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes( return false; NewRecipe = new VPWidenIntrinsicRecipe( *CI, getVectorIntrinsicIDForCall(CI, &TLI), - drop_end(Ingredient.operands()), CI->getType(), + drop_end(Ingredient.operands()), CI->getType(), *VPI, CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); } else if (auto *CI = dyn_cast(Inst)) { - NewRecipe = new VPWidenCastRecipe( - CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), *CI); + NewRecipe = + new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0), + CI->getType(), CI, {}, *VPI); } else { - NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands()); + NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI, + Ingredient.getDebugLoc()); } } @@ -1845,7 +1846,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF, // The vector region contains header phis for which we cannot remove the // loop region yet. auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()}, - Term->getDebugLoc()); + {}, Term->getDebugLoc()); ExitingVPBB->appendRecipe(BOC); } @@ -3815,15 +3816,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red, Ext0->getOpcode() == Ext1->getOpcode() && IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) { auto *NewExt0 = new VPWidenCastRecipe( - Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0, - *Ext0, Ext0->getDebugLoc()); + Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr, + *Ext0, *Ext0, Ext0->getDebugLoc()); NewExt0->insertBefore(Ext0); VPWidenCastRecipe *NewExt1 = NewExt0; if (Ext0 != Ext1) { NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0), - Ext->getResultType(), *Ext1, *Ext1, - Ext1->getDebugLoc()); + Ext->getResultType(), nullptr, *Ext1, + *Ext1, Ext1->getDebugLoc()); NewExt1->insertBefore(Ext1); } Mul->setOperand(0, NewExt0); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index e3bde8a47dcbc..a44a4f69c917b 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -23,6 +23,7 @@ namespace llvm { class InductionDescriptor; class Instruction; +class LoopVersioning; class PHINode; class ScalarEvolution; class PredicatedScalarEvolution; @@ -99,7 +100,7 @@ struct VPlanTransforms { /// >[ ] <-- original loop exit block(s), wrapped in VPIRBasicBlocks. LLVM_ABI_FOR_TEST static std::unique_ptr buildVPlan0(Loop *TheLoop, LoopInfo &LI, Type *InductionTy, DebugLoc IVDL, - PredicatedScalarEvolution &PSE); + PredicatedScalarEvolution &PSE, LoopVersioning *LVer = nullptr); /// Update \p Plan to account for all early exits. LLVM_ABI_FOR_TEST static void handleEarlyExits(VPlan &Plan, diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index ee7fa175ca918..125a68c5acbf4 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1009,7 +1009,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) { SmallVector Args; Args.push_back(Op1); Args.push_back(Op2); - VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end())); + VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc()); checkVPRecipeCastImpl(&WidenR); delete AI; @@ -1092,7 +1092,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) { IntegerType *Int64 = IntegerType::get(C, 64); auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64); VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1)); - VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast); + VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, Cast); checkVPRecipeCastImpl(&Recipe); delete Cast; @@ -1263,7 +1263,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { SmallVector Args; Args.push_back(Op1); Args.push_back(Op2); - VPWidenRecipe Recipe(*AI, make_range(Args.begin(), Args.end())); + VPWidenRecipe Recipe(*AI, Args, VPIRMetadata(), DebugLoc()); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1468,7 +1468,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) { Args.push_back(ExtVPV1); Args.push_back(ExtVPV2); VPWidenRecipe *WidenR = - new VPWidenRecipe(*AI, make_range(Args.begin(), Args.end())); + new VPWidenRecipe(*AI, Args, VPIRMetadata(), DebugLoc()); VPBB1->appendRecipe(WidenR); { From 3034e99010cf5e6e23abe9aa7dc86b2f10b3c362 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Sun, 16 Nov 2025 10:46:21 +0000 Subject: [PATCH 2/5] !fixup address comments, more uniform constructors. --- .../Vectorize/LoopVectorizationPlanner.h | 38 +++++------ .../Transforms/Vectorize/LoopVectorize.cpp | 5 +- llvm/lib/Transforms/Vectorize/VPlan.h | 63 ++++++++++--------- llvm/lib/Transforms/Vectorize/VPlanSLP.cpp | 2 +- .../Transforms/Vectorize/VPlanTransforms.cpp | 26 ++++---- 5 files changed, 70 insertions(+), 64 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index 1843338d69ff4..f533a47150a7b 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -63,10 +63,11 @@ class VPBuilder { } VPInstruction *createInstruction(unsigned Opcode, - ArrayRef Operands, DebugLoc DL, + ArrayRef Operands, + const VPIRMetadata &MD, DebugLoc DL, const Twine &Name = "") { return tryInsertInstruction( - new VPInstruction(Opcode, Operands, {}, DL, Name)); + new VPInstruction(Opcode, Operands, {}, MD, DL, Name)); } public: @@ -154,14 +155,14 @@ class VPBuilder { const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - VPInstruction *NewVPInst = - tryInsertInstruction(new VPInstruction(Opcode, Operands, MD, DL, Name)); + VPInstruction *NewVPInst = tryInsertInstruction( + new VPInstruction(Opcode, Operands, {}, MD, DL, Name)); NewVPInst->setUnderlyingValue(Inst); return NewVPInst; } VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, DebugLoc DL, const Twine &Name = "") { - return createInstruction(Opcode, Operands, DL, Name); + return createInstruction(Opcode, Operands, {}, DL, Name); } VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, const VPIRFlags &Flags, @@ -175,8 +176,8 @@ class VPBuilder { Type *ResultTy, const VPIRFlags &Flags = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return tryInsertInstruction( - new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name)); + return tryInsertInstruction(new VPInstructionWithType( + Opcode, Operands, ResultTy, Flags, {}, DL, Name)); } VPInstruction *createOverflowingOp( @@ -190,13 +191,14 @@ class VPBuilder { VPInstruction *createNot(VPValue *Operand, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return createInstruction(VPInstruction::Not, {Operand}, DL, Name); + return createInstruction(VPInstruction::Not, {Operand}, {}, DL, Name); } VPInstruction *createAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name); + return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, {}, DL, + Name); } VPInstruction *createOr(VPValue *LHS, VPValue *RHS, @@ -211,20 +213,18 @@ class VPBuilder { VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { - return tryInsertInstruction( - new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, {}, DL, Name)); + return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name); } VPInstruction * createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "", std::optional FMFs = std::nullopt) { - auto *Select = - FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - *FMFs, {}, DL, Name) - : new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal}, - {}, DL, Name); - return tryInsertInstruction(Select); + if (!FMFs) + return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL, + Name); + return tryInsertInstruction(new VPInstruction( + Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name)); } /// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A @@ -307,7 +307,7 @@ class VPBuilder { const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}) { return tryInsertInstruction( - new VPInstructionWithType(Opcode, Op, ResultTy, DL, Flags, Metadata)); + new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL)); } VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy, @@ -329,7 +329,7 @@ class VPBuilder { else if (Opcode == Instruction::ZExt) Flags = VPIRFlags::NonNegFlagsTy(false); return tryInsertInstruction( - new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags)); + new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags)); } VPScalarIVStepsRecipe * diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index fadc0918f264d..356d759b94799 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -8234,13 +8234,14 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, return new VPWidenGEPRecipe(cast(Instr), R->operands()); if (VPI->getOpcode() == Instruction::Select) - return new VPWidenSelectRecipe(*cast(Instr), R->operands()); + return new VPWidenSelectRecipe(*cast(Instr), R->operands(), + *VPI); if (Instruction::isCast(VPI->getOpcode())) { auto *CastR = cast(R); auto *CI = cast(Instr); return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0), - CastR->getResultType(), CI, *VPI, *VPI); + CastR->getResultType(), *CI, *VPI); } return tryToWiden(VPI); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index fea9c9b7aa5e3..42d9ccdb3c90c 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1120,7 +1120,7 @@ class LLVM_ABI_FOR_TEST VPInstruction : public VPRecipeWithIRFlags, public: VPInstruction(unsigned Opcode, ArrayRef Operands, - const VPIRFlags &Flags, const VPIRMetadata &MD = {}, + const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = ""); VP_CLASSOF_IMPL(VPDef::VPInstructionSC) @@ -1210,14 +1210,10 @@ class VPInstructionWithType : public VPInstruction { public: VPInstructionWithType(unsigned Opcode, ArrayRef Operands, - Type *ResultTy, const VPIRFlags &Flags, DebugLoc DL, + Type *ResultTy, const VPIRFlags &Flags = {}, + const VPIRMetadata &Metadata = {}, + DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") - : VPInstruction(Opcode, Operands, Flags, {}, DL, Name), - ResultTy(ResultTy) {} - - VPInstructionWithType(unsigned Opcode, ArrayRef Operands, - Type *ResultTy, DebugLoc DL, const VPIRFlags &Flags, - const VPIRMetadata &Metadata, const Twine &Name = "") : VPInstruction(Opcode, Operands, Flags, Metadata, DL, Name), ResultTy(ResultTy) {} @@ -1246,7 +1242,7 @@ class VPInstructionWithType : public VPInstruction { VPInstruction *clone() override { auto *New = new VPInstructionWithType(getOpcode(), operands(), getResultType(), - *this, getDebugLoc(), getName()); + *this, *this, getDebugLoc(), getName()); New->setUnderlyingValue(getUnderlyingValue()); return New; } @@ -1330,7 +1326,7 @@ class VPPhiAccessors { struct LLVM_ABI_FOR_TEST VPPhi : public VPInstruction, public VPPhiAccessors { VPPhi(ArrayRef Operands, DebugLoc DL, const Twine &Name = "") - : VPInstruction(Instruction::PHI, Operands, {}, DL, Name) {} + : VPInstruction(Instruction::PHI, Operands, {}, {}, DL, Name) {} static inline bool classof(const VPUser *U) { auto *VPI = dyn_cast(U); @@ -1476,10 +1472,8 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags, VPWidenRecipe(Instruction &I, ArrayRef Operands, const VPIRMetadata &Metadata, DebugLoc DL) - : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, VPIRFlags(I), DL), - VPIRMetadata(Metadata), Opcode(I.getOpcode()) { - setUnderlyingValue(&I); - } + : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), + VPIRMetadata(Metadata), Opcode(I.getOpcode()) {} ~VPWidenRecipe() override = default; @@ -1520,26 +1514,30 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - CastInst *UI = nullptr, const VPIRFlags &Flags = {}, + CastInst &UI, const VPIRMetadata &Metadata) + : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), + VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) { + assert(UI.getOpcode() == Opcode && + "opcode of underlying cast doesn't match"); + } + VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, + const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}, DebugLoc DL = DebugLoc::getUnknown()) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, - UI ? VPIRFlags(*UI) : Flags, - UI ? UI->getDebugLoc() : DL), + : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL), VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) { assert(flagsValidForOpcode(Opcode) && "Set flags not supported for the provided opcode"); - assert((!UI || UI->getOpcode() == Opcode) && - "opcode of underlying cast doesn't match"); - setUnderlyingValue(UI); } ~VPWidenCastRecipe() override = default; VPWidenCastRecipe *clone() override { - return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, - cast_or_null(getUnderlyingValue()), - *this, *this, getDebugLoc()); + auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this, + *this, getDebugLoc()); + if (auto *UV = getUnderlyingValue()) + New->setUnderlyingValue(UV); + return New; } VP_CLASSOF_IMPL(VPDef::VPWidenCastSC) @@ -1594,9 +1592,13 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, + const VPIRFlags &Flags = {}, + const VPIRMetadata &Metadata = {}, DebugLoc DL = DebugLoc::getUnknown()) - : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, DL), - VPIRMetadata(), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty) { + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags, + DL), + VPIRMetadata(Metadata), VectorIntrinsicID(VectorIntrinsicID), + ResultTy(Ty) { LLVMContext &Ctx = Ty->getContext(); AttributeSet Attrs = Intrinsic::getFnAttributes(Ctx, VectorIntrinsicID); MemoryEffects ME = Attrs.getMemoryEffects(); @@ -1615,7 +1617,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { operands(), ResultTy, *this, getDebugLoc()); return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy, - getDebugLoc()); + *this, *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenIntrinsicSC) @@ -1756,15 +1758,16 @@ class VPHistogramRecipe : public VPRecipeBase { /// instruction. struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { - VPWidenSelectRecipe(SelectInst &I, ArrayRef Operands) + VPWidenSelectRecipe(SelectInst &I, ArrayRef Operands, + VPIRMetadata &MD) : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I), - VPIRMetadata(I) {} + VPIRMetadata(MD) {} ~VPWidenSelectRecipe() override = default; VPWidenSelectRecipe *clone() override { return new VPWidenSelectRecipe(*cast(getUnderlyingInstr()), - operands()); + operands(), *this); } VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC) diff --git a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp index 3b8ff2515a3f7..3b5cc9fcb9820 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanSLP.cpp @@ -518,7 +518,7 @@ VPInstruction *VPlanSlp::buildGraph(ArrayRef Values) { assert(CombinedOperands.size() > 0 && "Need more some operands"); auto *Inst = cast(Values[0])->getUnderlyingInstr(); auto *VPI = - new VPInstruction(Opcode, CombinedOperands, {}, Inst->getDebugLoc()); + new VPInstruction(Opcode, CombinedOperands, {}, {}, Inst->getDebugLoc()); LLVM_DEBUG(dbgs() << "Create VPInstruction " << *VPI << " " << Values[0] << "\n"); diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 41481b207e8e2..89118b49bed44 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -109,11 +109,11 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes( drop_end(Ingredient.operands()), CI->getType(), *VPI, CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { - NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); + NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands(), *VPI); } else if (auto *CI = dyn_cast(Inst)) { NewRecipe = new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0), - CI->getType(), CI, {}, *VPI); + CI->getType(), *CI, *VPI); } else { NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI, Ingredient.getDebugLoc()); @@ -1706,8 +1706,9 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF, Ops.append({ALM, Plan.getOrAddLiveIn( ConstantInt::get(IntegerType::getInt64Ty(Ctx), VF.getKnownMinValue() * Part))}); - auto *Ext = new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops, - IntegerType::getInt1Ty(Ctx), DL); + auto *Ext = + new VPWidenIntrinsicRecipe(Intrinsic::vector_extract, Ops, + IntegerType::getInt1Ty(Ctx), {}, {}, DL); Extracts[Part] = Ext; Ext->insertAfter(ALM); } @@ -1846,7 +1847,7 @@ static bool simplifyBranchConditionForVFAndUF(VPlan &Plan, ElementCount BestVF, // The vector region contains header phis for which we cannot remove the // loop region yet. auto *BOC = new VPInstruction(VPInstruction::BranchOnCond, {Plan.getTrue()}, - {}, Term->getDebugLoc()); + {}, {}, Term->getDebugLoc()); ExitingVPBB->appendRecipe(BOC); } @@ -2680,13 +2681,13 @@ static VPRecipeBase *optimizeMaskToEVL(VPValue *HeaderMask, m_Select(m_Specific(HeaderMask), m_VPValue(LHS), m_VPValue(RHS)))) return new VPWidenIntrinsicRecipe( Intrinsic::vp_merge, {Plan->getTrue(), LHS, RHS, &EVL}, - TypeInfo.inferScalarType(LHS), CurRecipe.getDebugLoc()); + TypeInfo.inferScalarType(LHS), {}, {}, CurRecipe.getDebugLoc()); if (match(&CurRecipe, m_Select(m_RemoveMask(HeaderMask, Mask), m_VPValue(LHS), m_VPValue(RHS)))) return new VPWidenIntrinsicRecipe( Intrinsic::vp_merge, {Mask, LHS, RHS, &EVL}, - TypeInfo.inferScalarType(LHS), CurRecipe.getDebugLoc()); + TypeInfo.inferScalarType(LHS), {}, {}, CurRecipe.getDebugLoc()); return nullptr; } @@ -2754,7 +2755,8 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) { VPWidenIntrinsicRecipe *VPSplice = new VPWidenIntrinsicRecipe( Intrinsic::experimental_vp_splice, {V1, V2, Imm, Plan.getTrue(), PrevEVL, &EVL}, - TypeInfo.inferScalarType(R.getVPSingleValue()), R.getDebugLoc()); + TypeInfo.inferScalarType(R.getVPSingleValue()), {}, {}, + R.getDebugLoc()); VPSplice->insertBefore(&R); R.getVPSingleValue()->replaceAllUsesWith(VPSplice); ToErase.push_back(&R); @@ -3816,15 +3818,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red, Ext0->getOpcode() == Ext1->getOpcode() && IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) { auto *NewExt0 = new VPWidenCastRecipe( - Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr, - *Ext0, *Ext0, Ext0->getDebugLoc()); + Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0, + *Ext0, Ext0->getDebugLoc()); NewExt0->insertBefore(Ext0); VPWidenCastRecipe *NewExt1 = NewExt0; if (Ext0 != Ext1) { NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0), - Ext->getResultType(), nullptr, *Ext1, - *Ext1, Ext1->getDebugLoc()); + Ext->getResultType(), *Ext1, *Ext1, + Ext1->getDebugLoc()); NewExt1->insertBefore(Ext1); } Mul->setOperand(0, NewExt0); From 7cb092b2ba2a4c07ee0d4c06ff7f2b3e8c4bec0b Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 17 Nov 2025 11:18:00 +0000 Subject: [PATCH 3/5] !fixup remove VPIRMetadata(Instruction, LoopVersioning), fix unit test. --- llvm/lib/Transforms/Vectorize/VPlan.h | 7 +------ llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp | 12 ------------ llvm/unittests/Transforms/Vectorize/VPlanTest.cpp | 2 +- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 42d9ccdb3c90c..c81834e401726 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -65,7 +65,6 @@ class VPReplicateRecipe; class VPlanSlp; class Value; class LoopVectorizationCostModel; -class LoopVersioning; struct VPCostContext; @@ -958,10 +957,6 @@ class VPIRMetadata { /// \p I. VPIRMetadata(Instruction &I) { getMetadataToPropagate(&I, Metadata); } - /// Adds metatadata that can be preserved from the original instruction - /// \p I and noalias metadata guaranteed by runtime checks using \p LVer. - VPIRMetadata(Instruction &I, LoopVersioning *LVer); - /// Copy constructor for cloning. VPIRMetadata(const VPIRMetadata &Other) = default; @@ -1759,7 +1754,7 @@ class VPHistogramRecipe : public VPRecipeBase { struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { VPWidenSelectRecipe(SelectInst &I, ArrayRef Operands, - VPIRMetadata &MD) + const VPIRMetadata &MD = {}) : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I), VPIRMetadata(MD) {} diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index e2a8e495d5ed5..fca6554ad77c6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -36,7 +36,6 @@ #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/LoopUtils.h" -#include "llvm/Transforms/Utils/LoopVersioning.h" #include using namespace llvm; @@ -1674,17 +1673,6 @@ void VPIRPhi::printRecipe(raw_ostream &O, const Twine &Indent, } #endif -VPIRMetadata::VPIRMetadata(Instruction &I, LoopVersioning *LVer) - : VPIRMetadata(I) { - if (!LVer || !isa(&I)) - return; - const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(&I); - if (AliasScopeMD) - Metadata.emplace_back(LLVMContext::MD_alias_scope, AliasScopeMD); - if (NoAliasMD) - Metadata.emplace_back(LLVMContext::MD_noalias, NoAliasMD); -} - void VPIRMetadata::applyMetadata(Instruction &I) const { for (const auto &[Kind, Node] : Metadata) I.setMetadata(Kind, Node); diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 125a68c5acbf4..0e76c64f09f59 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1092,7 +1092,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) { IntegerType *Int64 = IntegerType::get(C, 64); auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64); VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1)); - VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, Cast); + VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast, {}); checkVPRecipeCastImpl(&Recipe); delete Cast; From 2906da8285d655119ec01dd62455720f8c218970 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 17 Nov 2025 14:57:02 +0000 Subject: [PATCH 4/5] !fixup addMetadata -> setMetadata --- llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index a3179aacaa6ba..556eaa37530eb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -241,9 +241,9 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, const auto &[AliasScopeMD, NoAliasMD] = LVer->getNoAliasMetadataFor(Inst); if (AliasScopeMD) - MD.addMetadata(LLVMContext::MD_alias_scope, AliasScopeMD); + MD.setMetadata(LLVMContext::MD_alias_scope, AliasScopeMD); if (NoAliasMD) - MD.addMetadata(LLVMContext::MD_noalias, NoAliasMD); + MD.setMetadata(LLVMContext::MD_noalias, NoAliasMD); } // Translate LLVM-IR operands into VPValue operands and set them in the From 546530c9308ee5ffb13d09861628e52b46292f09 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 17 Nov 2025 21:01:21 +0000 Subject: [PATCH 5/5] !fixup fix formatting --- llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 556eaa37530eb..612202d049774 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -254,12 +254,14 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, if (auto *CI = dyn_cast(Inst)) { NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0], - CI->getType(), CI->getDebugLoc(), {}, MD); + CI->getType(), CI->getDebugLoc(), + {}, MD); NewR->setUnderlyingValue(CI); } else { // Build VPInstruction for any arbitrary Instruction without specific // representation in VPlan. - NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD, Inst->getDebugLoc()); + NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD, + Inst->getDebugLoc()); } }