Skip to content

[VPlan] Populate and use VPIRMetadata from VPInstructions (NFC) #167253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 17, 2025
Merged

[VPlan] Populate and use VPIRMetadata from VPInstructions (NFC) #167253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
033f430
[VPlan] Populate and use VPIRMetadata from VPInstructions (NFC)
fhahn Nov 5, 2025
3034e99
!fixup address comments, more uniform constructors.
fhahn Nov 16, 2025
7cb092b
!fixup remove VPIRMetadata(Instruction, LoopVersioning), fix unit test.
fhahn Nov 17, 2025
2906da8
!fixup addMetadata -> setMetadata
fhahn Nov 17, 2025
546530c
!fixup fix formatting
fhahn Nov 17, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,11 @@ class VPBuilder {
}

VPInstruction *createInstruction(unsigned Opcode,
ArrayRef<VPValue *> Operands, DebugLoc DL,
ArrayRef<VPValue *> Operands,
const VPIRMetadata &MD, DebugLoc DL,
const Twine &Name = "") {
return tryInsertInstruction(new VPInstruction(Opcode, Operands, DL, Name));
return tryInsertInstruction(
new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
}

public:
Expand Down Expand Up @@ -150,17 +152,17 @@ class VPBuilder {
/// its underlying Instruction.
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
Instruction *Inst = nullptr,
const VPIRMetadata &MD = {},
Copy link
Contributor

@david-arm david-arm Nov 10, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe good to be consistent and also add a similar default parameter to createInstruction?

Copy link
Member

@rengolin rengolin Nov 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why not just add at the end?

Copy link
Contributor Author

@fhahn fhahn Nov 16, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Currently most helpers/constructors take DL/Name at the end, and the changes her try to keep things consistent.

I could add it to createInstruction and update all places that need. Some places do not support metatadata (like createNot, createAnd.

DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
DebugLoc DL = DebugLoc::getUnknown();
if (Inst)
DL = Inst->getDebugLoc();
VPInstruction *NewVPInst = createInstruction(Opcode, Operands, DL, Name);
VPInstruction *NewVPInst = tryInsertInstruction(
new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
NewVPInst->setUnderlyingValue(Inst);
return NewVPInst;
}
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
DebugLoc DL, const Twine &Name = "") {
return createInstruction(Opcode, Operands, DL, Name);
return createInstruction(Opcode, Operands, {}, DL, Name);
}
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags,
Expand All @@ -174,8 +176,8 @@ class VPBuilder {
Type *ResultTy, const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
return tryInsertInstruction(
new VPInstructionWithType(Opcode, Operands, ResultTy, Flags, DL, Name));
return tryInsertInstruction(new VPInstructionWithType(
Opcode, Operands, ResultTy, Flags, {}, DL, Name));
}

VPInstruction *createOverflowingOp(
Expand All @@ -189,13 +191,14 @@ class VPBuilder {
VPInstruction *createNot(VPValue *Operand,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
return createInstruction(VPInstruction::Not, {Operand}, DL, Name);
return createInstruction(VPInstruction::Not, {Operand}, {}, DL, Name);
}

VPInstruction *createAnd(VPValue *LHS, VPValue *RHS,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, DL, Name);
return createInstruction(Instruction::BinaryOps::And, {LHS, RHS}, {}, DL,
Name);
}

VPInstruction *createOr(VPValue *LHS, VPValue *RHS,
Expand All @@ -210,20 +213,18 @@ class VPBuilder {
VPInstruction *createLogicalAnd(VPValue *LHS, VPValue *RHS,
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
return tryInsertInstruction(
new VPInstruction(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name));
return createNaryOp(VPInstruction::LogicalAnd, {LHS, RHS}, DL, Name);
}

VPInstruction *
createSelect(VPValue *Cond, VPValue *TrueVal, VPValue *FalseVal,
DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "",
std::optional<FastMathFlags> FMFs = std::nullopt) {
auto *Select =
FMFs ? new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
*FMFs, {}, DL, Name)
: new VPInstruction(Instruction::Select, {Cond, TrueVal, FalseVal},
DL, Name);
return tryInsertInstruction(Select);
if (!FMFs)
return createNaryOp(Instruction::Select, {Cond, TrueVal, FalseVal}, DL,
Name);
return tryInsertInstruction(new VPInstruction(
Instruction::Select, {Cond, TrueVal, FalseVal}, *FMFs, {}, DL, Name));
}

/// Create a new ICmp VPInstruction with predicate \p Pred and operands \p A
Expand Down Expand Up @@ -306,7 +307,7 @@ class VPBuilder {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}) {
return tryInsertInstruction(
new VPInstructionWithType(Opcode, Op, ResultTy, DL, Flags, Metadata));
new VPInstructionWithType(Opcode, Op, ResultTy, Flags, Metadata, DL));
}

VPValue *createScalarZExtOrTrunc(VPValue *Op, Type *ResultTy, Type *SrcTy,
Expand Down
39 changes: 20 additions & 19 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7616,14 +7616,13 @@ VPWidenMemoryRecipe *VPRecipeBuilder::tryToWidenMemory(VPInstruction *VPI,
}
if (VPI->getOpcode() == Instruction::Load) {
auto *Load = cast<LoadInst>(I);
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse,
VPIRMetadata(*Load, LVer), I->getDebugLoc());
return new VPWidenLoadRecipe(*Load, Ptr, Mask, Consecutive, Reverse, *VPI,
VPI->getDebugLoc());
}

StoreInst *Store = cast<StoreInst>(I);
return new VPWidenStoreRecipe(*Store, Ptr, VPI->getOperand(0), Mask,
Consecutive, Reverse,
VPIRMetadata(*Store, LVer), VPI->getDebugLoc());
Consecutive, Reverse, *VPI, VPI->getDebugLoc());
}

/// Creates a VPWidenIntOrFpInductionRecipe for \p PhiR. If needed, it will
Expand Down Expand Up @@ -7751,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
},
Range);
if (ShouldUseVectorIntrinsic)
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(),
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI,
VPI->getDebugLoc());

Function *Variant = nullptr;
Expand Down Expand Up @@ -7843,7 +7842,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
return new VPWidenRecipe(*I, Ops);
return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc());
}
[[fallthrough]];
}
Expand Down Expand Up @@ -7889,15 +7888,15 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
return new VPWidenRecipe(*I, NewOps);
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(VPI->operands());
auto *EVI = cast<ExtractValueInst>(I);
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getConstantInt(32, Idx));
return new VPWidenRecipe(*I, NewOps);
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
}
};
}
Expand Down Expand Up @@ -7981,8 +7980,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
assert((Range.Start.isScalar() || !IsUniform || !IsPredicated ||
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe = new VPReplicateRecipe(I, VPI->operands(), IsUniform,
BlockInMask, VPIRMetadata(*I, LVer));
auto *Recipe =
new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI);
return Recipe;
}

Expand Down Expand Up @@ -8235,13 +8234,14 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());

if (VPI->getOpcode() == Instruction::Select)
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands());
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands(),
*VPI);

if (Instruction::isCast(VPI->getOpcode())) {
auto *CastR = cast<VPInstructionWithType>(R);
auto *CI = cast<CastInst>(Instr);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
CastR->getResultType(), *CI);
CastR->getResultType(), *CI, *VPI);
}

return tryToWiden(VPI);
Expand Down Expand Up @@ -8269,7 +8269,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
BinOp = new VPWidenRecipe(*ReductionI, Ops);
BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(),
ReductionI->getDebugLoc());
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
Expand Down Expand Up @@ -8302,7 +8303,7 @@ void LoopVectorizationPlanner::buildVPlansWithVPRecipes(ElementCount MinVF,
// candidates built later for specific VF ranges.
auto VPlan0 = VPlanTransforms::buildVPlan0(
OrigLoop, *LI, Legal->getWidestInductionType(),
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE);
getDebugLocFromInstOrOperands(Legal->getPrimaryInduction()), PSE, &LVer);

auto MaxVFTimes2 = MaxVF * 2;
for (ElementCount VF = MinVF; ElementCount::isKnownLT(VF, MaxVFTimes2);) {
Expand Down Expand Up @@ -8408,7 +8409,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
// VPInstructions in the loop.
// ---------------------------------------------------------------------------
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
Builder, BlockMaskCache, LVer);
Builder, BlockMaskCache);
// TODO: Handle partial reductions with EVL tail folding.
if (!CM.foldTailWithEVL())
RecipeBuilder.collectScaledReductions(Range);
Expand Down Expand Up @@ -8453,9 +8454,9 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *Recipe =
new VPReplicateRecipe(SI, R.operands(), true /* IsUniform */,
nullptr /*Mask*/, VPIRMetadata(*SI, LVer));
auto *Recipe = new VPReplicateRecipe(
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
*cast<VPInstruction>(SingleDef));
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
Expand Down Expand Up @@ -8606,7 +8607,7 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlan(VFRange &Range) {
// addScalarResumePhis.
DenseMap<VPBasicBlock *, VPValue *> BlockMaskCache;
VPRecipeBuilder RecipeBuilder(*Plan, OrigLoop, TLI, &TTI, Legal, CM, PSE,
Builder, BlockMaskCache, nullptr /*LVer*/);
Builder, BlockMaskCache);
for (auto &R : Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
if (isa<VPCanonicalIVPHIRecipe>(&R))
continue;
Expand Down
10 changes: 2 additions & 8 deletions llvm/lib/Transforms/Vectorize/VPRecipeBuilder.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,6 @@ class VPRecipeBuilder {
/// A mapping of partial reduction exit instructions to their scaling factor.
DenseMap<const Instruction *, unsigned> ScaledReductionMap;

/// Loop versioning instance for getting noalias metadata guaranteed by
/// runtime checks.
LoopVersioning *LVer;

/// Check if \p I can be widened at the start of \p Range and possibly
/// decrease the range such that the returned value holds for the entire \p
/// Range. The function should not be called for memory instructions or calls.
Expand Down Expand Up @@ -144,11 +140,9 @@ class VPRecipeBuilder {
LoopVectorizationLegality *Legal,
LoopVectorizationCostModel &CM,
PredicatedScalarEvolution &PSE, VPBuilder &Builder,
DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache,
LoopVersioning *LVer)
DenseMap<VPBasicBlock *, VPValue *> &BlockMaskCache)
: Plan(Plan), OrigLoop(OrigLoop), TLI(TLI), TTI(TTI), Legal(Legal),
CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache),
LVer(LVer) {}
CM(CM), PSE(PSE), Builder(Builder), BlockMaskCache(BlockMaskCache) {}

std::optional<unsigned> getScalingForReduction(const Instruction *ExitInst) {
auto It = ScaledReductionMap.find(ExitInst);
Expand Down
Loading
Loading