From fb3bb395cab5996ee084a693d07204c947cf2978 Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Mon, 17 Nov 2025 21:55:07 +0000 Subject: [PATCH 1/2] [VPLan] Populate and use VPIRFlags from initial VPInstruction (NFC) Update VPlan to populate VPIRFlags during VPInstruction construction and use it when creating widened recipes, instead of constructing VPIRFlags from the underlying IR instruction each time. The VPRecipeWithIRFlags constructor taking an underlying instruction and setting the flags based on it has been removed. This centralizes initial VPIRFlags creation and ensures flags are consistently available throughout VPlan transformations and makes sure we don't accidentally re-add flags from the underlying instruction that already got dropped during transformations. Follow-up to https://github.com/llvm/llvm-project/pull/167253, which did the same for VPIRMetadata. --- .../Vectorize/LoopVectorizationPlanner.h | 5 +- .../Transforms/Vectorize/LoopVectorize.cpp | 35 ++++--- llvm/lib/Transforms/Vectorize/VPlan.h | 93 ++++++++++--------- .../Vectorize/VPlanConstruction.cpp | 11 ++- .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 16 ++-- .../Transforms/Vectorize/VPlanTransforms.cpp | 50 +++++----- llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp | 6 +- .../vplan-printing-outer-loop.ll | 14 +-- .../Transforms/Vectorize/VPlanHCFGTest.cpp | 10 +- .../Transforms/Vectorize/VPlanTest.cpp | 15 ++- 10 files changed, 135 insertions(+), 120 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h index f533a47150a7b..741392247c0d6 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h +++ b/llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h @@ -152,11 +152,12 @@ class VPBuilder { /// its underlying Instruction. VPInstruction *createNaryOp(unsigned Opcode, ArrayRef Operands, Instruction *Inst = nullptr, + const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown(), const Twine &Name = "") { VPInstruction *NewVPInst = tryInsertInstruction( - new VPInstruction(Opcode, Operands, {}, MD, DL, Name)); + new VPInstruction(Opcode, Operands, Flags, MD, DL, Name)); NewVPInst->setUnderlyingValue(Inst); return NewVPInst; } @@ -329,7 +330,7 @@ class VPBuilder { else if (Opcode == Instruction::ZExt) Flags = VPIRFlags::NonNegFlagsTy(false); return tryInsertInstruction( - new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags)); + new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags)); } VPScalarIVStepsRecipe * diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 356d759b94799..c680b6fca84cd 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7750,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI, }, Range); if (ShouldUseVectorIntrinsic) - return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, + return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, *VPI, VPI->getDebugLoc()); Function *Variant = nullptr; @@ -7804,7 +7804,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI, } Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1)); - return new VPWidenCallRecipe(CI, Variant, Ops, VPI->getDebugLoc()); + return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI, + VPI->getDebugLoc()); } return nullptr; @@ -7842,7 +7843,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { auto *SafeRHS = Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc()); Ops[1] = SafeRHS; - return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc()); + return new VPWidenRecipe(*I, Ops, *VPI, *VPI, VPI->getDebugLoc()); } [[fallthrough]]; } @@ -7888,7 +7889,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { // For other binops, the legacy cost model only checks the second operand. NewOps[1] = GetConstantViaSCEV(NewOps[1]); } - return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc()); + return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc()); } case Instruction::ExtractValue: { SmallVector NewOps(VPI->operands()); @@ -7896,7 +7897,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) { assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index"); unsigned Idx = EVI->getIndices()[0]; NewOps.push_back(Plan.getConstantInt(32, Idx)); - return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc()); + return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc()); } }; } @@ -7981,7 +7982,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI, (Range.Start.isScalable() && isa(I))) && "Should not predicate a uniform recipe"); auto *Recipe = - new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI); + new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI, + *VPI, VPI->getDebugLoc()); return Recipe; } @@ -8231,17 +8233,19 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R, return nullptr; if (VPI->getOpcode() == Instruction::GetElementPtr) - return new VPWidenGEPRecipe(cast(Instr), R->operands()); + return new VPWidenGEPRecipe(cast(Instr), R->operands(), + *VPI, VPI->getDebugLoc()); if (VPI->getOpcode() == Instruction::Select) - return new VPWidenSelectRecipe(*cast(Instr), R->operands(), - *VPI); + return new VPWidenSelectRecipe(cast(Instr), R->operands(), *VPI, + *VPI, VPI->getDebugLoc()); if (Instruction::isCast(VPI->getOpcode())) { - auto *CastR = cast(R); auto *CI = cast(Instr); + auto *CastR = cast(VPI); return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0), - CastR->getResultType(), *CI, *VPI); + CastR->getResultType(), CI, *VPI, *VPI, + VPI->getDebugLoc()); } return tryToWiden(VPI); @@ -8269,8 +8273,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction, SmallVector Ops; Ops.push_back(Plan.getOrAddLiveIn(Zero)); Ops.push_back(BinOp); - BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(), - ReductionI->getDebugLoc()); + BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRFlags(*ReductionI), + VPIRMetadata(), ReductionI->getDebugLoc()); Builder.insert(BinOp->getDefiningRecipe()); ReductionOpcode = Instruction::Add; } @@ -8454,9 +8458,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes( Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) { // Only create recipe for the final invariant store of the reduction. if (Legal->isInvariantStoreOfReduction(SI)) { + auto *VPI = cast(SingleDef); auto *Recipe = new VPReplicateRecipe( - SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, - *cast(SingleDef)); + SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, *VPI, + *VPI, VPI->getDebugLoc()); Recipe->insertBefore(*MiddleVPBB, MBIP); } R.eraseFromParent(); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index c81834e401726..094c9668b54ef 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -881,14 +881,6 @@ class VPIRFlags { /// A pure-virtual common base class for recipes defining a single VPValue and /// using IR flags. struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags { - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - DebugLoc DL = DebugLoc::getUnknown()) - : VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {} - - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, - Instruction &I) - : VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {} - VPRecipeWithIRFlags(const unsigned char SC, ArrayRef Operands, const VPIRFlags &Flags, DebugLoc DL = DebugLoc::getUnknown()) @@ -1466,9 +1458,12 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags, VPIRMetadata(Metadata), Opcode(Opcode) {} VPWidenRecipe(Instruction &I, ArrayRef Operands, - const VPIRMetadata &Metadata, DebugLoc DL) - : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I), - VPIRMetadata(Metadata), Opcode(I.getOpcode()) {} + const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}, + DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL), + VPIRMetadata(Metadata), Opcode(I.getOpcode()) { + setUnderlyingValue(&I); + } ~VPWidenRecipe() override = default; @@ -1509,30 +1504,23 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - CastInst &UI, const VPIRMetadata &Metadata) - : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI), - VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) { - assert(UI.getOpcode() == Opcode && - "opcode of underlying cast doesn't match"); - } - VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, - const VPIRFlags &Flags = {}, + CastInst *CI = nullptr, const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {}, DebugLoc DL = DebugLoc::getUnknown()) : VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL), VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) { assert(flagsValidForOpcode(Opcode) && "Set flags not supported for the provided opcode"); + setUnderlyingValue(CI); } ~VPWidenCastRecipe() override = default; VPWidenCastRecipe *clone() override { - auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this, - *this, getDebugLoc()); - if (auto *UV = getUnderlyingValue()) - New->setUnderlyingValue(UV); - return New; + return new VPWidenCastRecipe( + Opcode, getOperand(0), ResultTy, + dyn_cast_or_null(getUnderlyingValue()), *this, *this, + getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCastSC) @@ -1577,13 +1565,17 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { public: VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, + const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {}, DebugLoc DL = DebugLoc::getUnknown()) - : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI), + : VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags, + DL), VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty), MayReadFromMemory(CI.mayReadFromMemory()), MayWriteToMemory(CI.mayWriteToMemory()), - MayHaveSideEffects(CI.mayHaveSideEffects()) {} + MayHaveSideEffects(CI.mayHaveSideEffects()) { + setUnderlyingValue(&CI); + } VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID, ArrayRef CallArguments, Type *Ty, @@ -1609,7 +1601,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { VPWidenIntrinsicRecipe *clone() override { if (Value *CI = getUnderlyingValue()) return new VPWidenIntrinsicRecipe(*cast(CI), VectorIntrinsicID, - operands(), ResultTy, *this, + operands(), ResultTy, *this, *this, getDebugLoc()); return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy, *this, *this, getDebugLoc()); @@ -1663,10 +1655,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags, public: VPWidenCallRecipe(Value *UV, Function *Variant, ArrayRef CallArguments, - DebugLoc DL = DebugLoc::getUnknown()) - : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, - *cast(UV)), - VPIRMetadata(*cast(UV)), Variant(Variant) { + const VPIRFlags &Flags = {}, + const VPIRMetadata &Metadata = {}, DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL), + VPIRMetadata(Metadata), Variant(Variant) { + setUnderlyingValue(UV); assert( isa(getOperand(getNumOperands() - 1)->getLiveInIRValue()) && "last operand must be the called function"); @@ -1676,7 +1669,7 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags, VPWidenCallRecipe *clone() override { return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(), - getDebugLoc()); + *this, *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCallSC) @@ -1753,16 +1746,19 @@ class VPHistogramRecipe : public VPRecipeBase { /// instruction. struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { - VPWidenSelectRecipe(SelectInst &I, ArrayRef Operands, - const VPIRMetadata &MD = {}) - : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I), - VPIRMetadata(MD) {} + VPWidenSelectRecipe(SelectInst *SI, ArrayRef Operands, + const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {}, + DebugLoc DL = {}) + : VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, Flags, DL), + VPIRMetadata(MD) { + setUnderlyingValue(SI); + } ~VPWidenSelectRecipe() override = default; VPWidenSelectRecipe *clone() override { - return new VPWidenSelectRecipe(*cast(getUnderlyingInstr()), - operands(), *this); + return new VPWidenSelectRecipe(cast(getUnderlyingInstr()), + operands(), *this, *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC) @@ -1814,9 +1810,12 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags { } public: - VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef Operands) - : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP), + VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef Operands, + const VPIRFlags &Flags = {}, + DebugLoc DL = DebugLoc::getUnknown()) + : VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, Flags, DL), SourceElementTy(GEP->getSourceElementType()) { + setUnderlyingValue(GEP); SmallVector> Metadata; (void)Metadata; getMetadataToPropagate(GEP, Metadata); @@ -1827,7 +1826,7 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags { VPWidenGEPRecipe *clone() override { return new VPWidenGEPRecipe(cast(getUnderlyingInstr()), - operands()); + operands(), *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC) @@ -2921,10 +2920,12 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags, public: VPReplicateRecipe(Instruction *I, ArrayRef Operands, bool IsSingleScalar, VPValue *Mask = nullptr, - VPIRMetadata Metadata = {}) - : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I), + const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {}, + DebugLoc DL = DebugLoc::getUnknown()) + : VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, Flags, DL), VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar), IsPredicated(Mask) { + setUnderlyingValue(I); if (Mask) addOperand(Mask); } @@ -2932,9 +2933,9 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags, ~VPReplicateRecipe() override = default; VPReplicateRecipe *clone() override { - auto *Copy = - new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsSingleScalar, - isPredicated() ? getMask() : nullptr, *this); + auto *Copy = new VPReplicateRecipe( + getUnderlyingInstr(), operands(), IsSingleScalar, + isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc()); Copy->transferFlags(*this); return Copy; } diff --git a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp index 612202d049774..dbbde1cafa9f2 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp @@ -190,7 +190,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, // recipes. if (Br->isConditional()) { VPValue *Cond = getOrCreateVPOperand(Br->getCondition()); - VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, + VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, {}, VPIRMetadata(*Inst), Inst->getDebugLoc()); } @@ -205,7 +205,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, SmallVector Ops = {getOrCreateVPOperand(SI->getCondition())}; for (auto Case : SI->cases()) Ops.push_back(getOrCreateVPOperand(Case.getCaseValue())); - VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, + VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, {}, VPIRMetadata(*Inst), Inst->getDebugLoc()); continue; } @@ -255,13 +255,14 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB, if (auto *CI = dyn_cast(Inst)) { NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0], CI->getType(), CI->getDebugLoc(), - {}, MD); + VPIRFlags(*CI), MD); NewR->setUnderlyingValue(CI); } else { // Build VPInstruction for any arbitrary Instruction without specific // representation in VPlan. - NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD, - Inst->getDebugLoc()); + NewR = + VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, + VPIRFlags(*Inst), MD, Inst->getDebugLoc()); } } diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index fca6554ad77c6..ef36e29aaa5c4 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -2056,24 +2056,26 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const { switch (OpType) { case OperationType::OverflowingBinOp: return Opcode == Instruction::Add || Opcode == Instruction::Sub || - Opcode == Instruction::Mul || + Opcode == Instruction::Mul || Opcode == Instruction::Shl || Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart; case OperationType::Trunc: return Opcode == Instruction::Trunc; case OperationType::DisjointOp: return Opcode == Instruction::Or; case OperationType::PossiblyExactOp: - return Opcode == Instruction::AShr; + return Opcode == Instruction::AShr || Opcode == Instruction::LShr || + Opcode == Instruction::UDiv || Opcode == Instruction::SDiv; case OperationType::GEPOp: return Opcode == Instruction::GetElementPtr || Opcode == VPInstruction::PtrAdd || Opcode == VPInstruction::WidePtrAdd; case OperationType::FPMathOp: - return Opcode == Instruction::FAdd || Opcode == Instruction::FMul || - Opcode == Instruction::FSub || Opcode == Instruction::FNeg || - Opcode == Instruction::FDiv || Opcode == Instruction::FRem || - Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc || - Opcode == Instruction::FCmp || Opcode == Instruction::Select || + return Opcode == Instruction::Call || Opcode == Instruction::FAdd || + Opcode == Instruction::FMul || Opcode == Instruction::FSub || + Opcode == Instruction::FNeg || Opcode == Instruction::FDiv || + Opcode == Instruction::FRem || Opcode == Instruction::FPExt || + Opcode == Instruction::FPTrunc || Opcode == Instruction::FCmp || + Opcode == Instruction::Select || Opcode == VPInstruction::WideIVStep || Opcode == VPInstruction::ReductionStartVector || Opcode == VPInstruction::ComputeReductionResult; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 89118b49bed44..e934240576ea6 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -99,24 +99,26 @@ bool VPlanTransforms::tryToConvertVPInstructionsToVPRecipes( nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/, *VPI, Ingredient.getDebugLoc()); } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { - NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); + NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands(), *VPI, + Ingredient.getDebugLoc()); } else if (CallInst *CI = dyn_cast(Inst)) { Intrinsic::ID VectorID = getVectorIntrinsicIDForCall(CI, &TLI); if (VectorID == Intrinsic::not_intrinsic) return false; NewRecipe = new VPWidenIntrinsicRecipe( *CI, getVectorIntrinsicIDForCall(CI, &TLI), - drop_end(Ingredient.operands()), CI->getType(), *VPI, - CI->getDebugLoc()); + drop_end(Ingredient.operands()), CI->getType(), VPIRFlags(*CI), + *VPI, CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { - NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands(), *VPI); + NewRecipe = new VPWidenSelectRecipe(SI, Ingredient.operands(), *VPI, + *VPI, Ingredient.getDebugLoc()); } else if (auto *CI = dyn_cast(Inst)) { - NewRecipe = - new VPWidenCastRecipe(CI->getOpcode(), Ingredient.getOperand(0), - CI->getType(), *CI, *VPI); + NewRecipe = new VPWidenCastRecipe( + CI->getOpcode(), Ingredient.getOperand(0), CI->getType(), CI, + VPIRFlags(*CI), VPIRMetadata(*CI)); } else { NewRecipe = new VPWidenRecipe(*Inst, Ingredient.operands(), *VPI, - Ingredient.getDebugLoc()); + *VPI, Ingredient.getDebugLoc()); } } @@ -221,7 +223,8 @@ static bool sinkScalarOperands(VPlan &Plan) { // then cloning should be sufficient here. Instruction *I = SinkCandidate->getUnderlyingInstr(); Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true, - nullptr /*Mask*/, *SinkCandidateRepR); + nullptr /*Mask*/, *SinkCandidateRepR, + *SinkCandidateRepR); // TODO: add ".cloned" suffix to name of Clone's VPValue. } else { Clone = SinkCandidate->clone(); @@ -380,7 +383,8 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe, // mask but in the replicate region. auto *RecipeWithoutMask = new VPReplicateRecipe( PredRecipe->getUnderlyingInstr(), drop_end(PredRecipe->operands()), - PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe); + PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe, *PredRecipe, + PredRecipe->getDebugLoc()); auto *Pred = Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask); @@ -686,7 +690,7 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { // analysis. auto Users = collectUsersRecursively(PhiR); for (VPUser *U : reverse(Users)) { - auto *Def = dyn_cast(U); + auto *Def = dyn_cast(U); auto *RepR = dyn_cast(U); // Skip recipes that shouldn't be narrowed. if (!Def || !isa(Def) || @@ -699,7 +703,8 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) { continue; auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(), - Def->operands(), /*IsUniform*/ true); + Def->operands(), /*IsUniform*/ true, + /*Mask*/ nullptr, /*Flags*/ *Def); Clone->insertAfter(Def); Def->replaceAllUsesWith(Clone); } @@ -1418,12 +1423,13 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { if (RepR && (RepR->isSingleScalar() || RepR->isPredicated())) continue; - auto *RepOrWidenR = cast(&R); + auto *RepOrWidenR = cast(&R); if (RepR && isa(RepR->getUnderlyingInstr()) && vputils::isSingleScalar(RepR->getOperand(1))) { auto *Clone = new VPReplicateRecipe( RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(), - true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Metadata*/); + true /*IsSingleScalar*/, nullptr /*Mask*/, *RepR /*Flags*/, + *RepR /*Metadata*/, RepR->getDebugLoc()); Clone->insertBefore(RepOrWidenR); unsigned ExtractOpc = vputils::isUniformAcrossVFsAndUFs(RepR->getOperand(1)) @@ -1464,9 +1470,9 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { })) continue; - auto *Clone = new VPReplicateRecipe(RepOrWidenR->getUnderlyingInstr(), - RepOrWidenR->operands(), - true /*IsSingleScalar*/); + auto *Clone = new VPReplicateRecipe( + RepOrWidenR->getUnderlyingInstr(), RepOrWidenR->operands(), + true /*IsSingleScalar*/, nullptr, *RepOrWidenR); Clone->insertBefore(RepOrWidenR); RepOrWidenR->replaceAllUsesWith(Clone); if (isDeadRecipe(*RepOrWidenR)) @@ -3818,15 +3824,15 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red, Ext0->getOpcode() == Ext1->getOpcode() && IsMulAccValidAndClampRange(Mul, Ext0, Ext1, Ext) && Mul->hasOneUse()) { auto *NewExt0 = new VPWidenCastRecipe( - Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), *Ext0, - *Ext0, Ext0->getDebugLoc()); + Ext0->getOpcode(), Ext0->getOperand(0), Ext->getResultType(), nullptr, + *Ext0, *Ext0, Ext0->getDebugLoc()); NewExt0->insertBefore(Ext0); VPWidenCastRecipe *NewExt1 = NewExt0; if (Ext0 != Ext1) { NewExt1 = new VPWidenCastRecipe(Ext1->getOpcode(), Ext1->getOperand(0), - Ext->getResultType(), *Ext1, *Ext1, - Ext1->getDebugLoc()); + Ext->getResultType(), nullptr, *Ext1, + *Ext1, Ext1->getDebugLoc()); NewExt1->insertBefore(Ext1); } Mul->setOperand(0, NewExt0); @@ -4299,7 +4305,7 @@ narrowInterleaveGroupOp(VPValue *V, SmallPtrSetImpl &NarrowedOps) { // process one original iteration. auto *N = new VPReplicateRecipe(&WideLoad->getIngredient(), {PtrOp}, /*IsUniform*/ true, - /*Mask*/ nullptr, *WideLoad); + /*Mask*/ nullptr, {}, *WideLoad); N->insertBefore(WideLoad); NarrowedOps.insert(N); return N; diff --git a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp index d4b8b72beb942..d76d2ed5f1c76 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp @@ -518,9 +518,9 @@ cloneForLane(VPlan &Plan, VPBuilder &Builder, Type *IdxTy, // TODO: have cloning of replicate recipes also provide the desired result // coupled with setting its operands to NewOps (deriving IsSingleScalar and // Mask from the operands?) - New = - new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps, - /*IsSingleScalar=*/true, /*Mask=*/nullptr, *RepR); + New = new VPReplicateRecipe(RepR->getUnderlyingInstr(), NewOps, + /*IsSingleScalar=*/true, /*Mask=*/nullptr, + *RepR, *RepR, RepR->getDebugLoc()); } else { assert(isa(DefR) && "DefR must be a VPReplicateRecipe or VPInstruction"); diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll index 20676f3702294..10c265519952b 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing-outer-loop.ll @@ -14,23 +14,23 @@ define void @foo(i64 %n) { ; CHECK-EMPTY: ; CHECK-NEXT: outer.header: ; CHECK-NEXT: EMIT-SCALAR ir<%outer.iv> = phi [ ir<%outer.iv.next>, outer.latch ], [ ir<0>, ir-bb ] -; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr ir<@arr2>, ir<0>, ir<%outer.iv> +; CHECK-NEXT: EMIT ir<%gep.1> = getelementptr inbounds ir<@arr2>, ir<0>, ir<%outer.iv> ; CHECK-NEXT: EMIT store ir<%outer.iv>, ir<%gep.1> -; CHECK-NEXT: EMIT ir<%add> = add ir<%outer.iv>, ir<%n> +; CHECK-NEXT: EMIT ir<%add> = add nsw ir<%outer.iv>, ir<%n> ; CHECK-NEXT: Successor(s): inner ; CHECK-EMPTY: ; CHECK-NEXT: inner: ; CHECK-NEXT: EMIT-SCALAR ir<%inner.iv> = phi [ ir<%inner.iv.next>, inner ], [ ir<0>, outer.header ] -; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv> +; CHECK-NEXT: EMIT ir<%gep.2> = getelementptr inbounds ir<@arr>, ir<0>, ir<%inner.iv>, ir<%outer.iv> ; CHECK-NEXT: EMIT store ir<%add>, ir<%gep.2> -; CHECK-NEXT: EMIT ir<%inner.iv.next> = add ir<%inner.iv>, ir<1> -; CHECK-NEXT: EMIT ir<%inner.ec> = icmp ir<%inner.iv.next>, ir<8> +; CHECK-NEXT: EMIT ir<%inner.iv.next> = add nuw nsw ir<%inner.iv>, ir<1> +; CHECK-NEXT: EMIT ir<%inner.ec> = icmp eq ir<%inner.iv.next>, ir<8> ; CHECK-NEXT: EMIT branch-on-cond ir<%inner.ec> ; CHECK-NEXT: Successor(s): outer.latch, inner ; CHECK-EMPTY: ; CHECK-NEXT: outer.latch: -; CHECK-NEXT: EMIT ir<%outer.iv.next> = add ir<%outer.iv>, ir<1> -; CHECK-NEXT: EMIT ir<%outer.ec> = icmp ir<%outer.iv.next>, ir<8> +; CHECK-NEXT: EMIT ir<%outer.iv.next> = add nuw nsw ir<%outer.iv>, ir<1> +; CHECK-NEXT: EMIT ir<%outer.ec> = icmp eq ir<%outer.iv.next>, ir<8> ; CHECK-NEXT: EMIT branch-on-cond ir<%outer.ec> ; CHECK-NEXT: Successor(s): ir-bb, outer.header ; CHECK-EMPTY: diff --git a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp index b99d656c5c50f..5742df2aa3c53 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanHCFGTest.cpp @@ -139,12 +139,12 @@ compound=true "vector.body:\l" + " EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" + " EMIT-SCALAR ir\<%indvars.iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%indvars.iv.next\>, vector.body ]\l" + - " EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%indvars.iv\>\l" + + " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%indvars.iv\>\l" + " EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" + " EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" + " EMIT store ir\<%res\>, ir\<%arr.idx\>\l" + " EMIT ir\<%indvars.iv.next\> = add ir\<%indvars.iv\>, ir\<1\>\l" + - " EMIT ir\<%exitcond\> = icmp ir\<%indvars.iv.next\>, ir\<%N\>\l" + + " EMIT ir\<%exitcond\> = icmp ne ir\<%indvars.iv.next\>, ir\<%N\>\l" + " EMIT vp\<%3\> = not ir\<%exitcond\>\l" + " EMIT vp\<%index.next\> = add nuw vp\<%2\>, vp\<%0\>\l" + " EMIT branch-on-count vp\<%index.next\>, vp\<%1\>\l" + @@ -305,9 +305,9 @@ compound=true "vector.body:\l" + " EMIT vp\<%2\> = CANONICAL-INDUCTION ir\<0\>, vp\<%index.next\>\l" + " EMIT-SCALAR ir\<%iv\> = phi [ ir\<0\>, vector.ph ], [ ir\<%iv.next\>, loop.latch ]\l" + - " EMIT ir\<%arr.idx\> = getelementptr ir\<%A\>, ir\<%iv\>\l" + + " EMIT ir\<%arr.idx\> = getelementptr inbounds ir\<%A\>, ir\<%iv\>\l" + " EMIT ir\<%l1\> = load ir\<%arr.idx\>\l" + - " EMIT ir\<%c\> = icmp ir\<%l1\>, ir\<0\>\l" + + " EMIT ir\<%c\> = icmp eq ir\<%l1\>, ir\<0\>\l" + "Successor(s): loop.latch\l" ] N4 -> N6 [ label=""] @@ -316,7 +316,7 @@ compound=true " EMIT ir\<%res\> = add ir\<%l1\>, ir\<10\>\l" + " EMIT store ir\<%res\>, ir\<%arr.idx\>\l" + " EMIT ir\<%iv.next\> = add ir\<%iv\>, ir\<1\>\l" + - " EMIT ir\<%exitcond\> = icmp ir\<%iv.next\>, ir\<%N\>\l" + + " EMIT ir\<%exitcond\> = icmp ne ir\<%iv.next\>, ir\<%N\>\l" + " EMIT vp\<%3\> = not ir\<%exitcond\>\l" + " EMIT vp\<%index.next\> = add nuw vp\<%2\>, vp\<%0\>\l" + " EMIT branch-on-count vp\<%index.next\>, vp\<%1\>\l" + diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 0e76c64f09f59..78874208d10ee 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -1009,7 +1009,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) { SmallVector Args; Args.push_back(Op1); Args.push_back(Op2); - VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc()); + VPWidenRecipe WidenR(*AI, Args); checkVPRecipeCastImpl(&WidenR); delete AI; @@ -1053,7 +1053,7 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) { Args.push_back(Op1); Args.push_back(Op2); Args.push_back(Op3); - VPWidenSelectRecipe WidenSelectR(*SelectI, + VPWidenSelectRecipe WidenSelectR(SelectI, make_range(Args.begin(), Args.end())); checkVPRecipeCastImpl(&WidenSelectR); @@ -1092,7 +1092,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) { IntegerType *Int64 = IntegerType::get(C, 64); auto *Cast = CastInst::CreateZExtOrBitCast(PoisonValue::get(Int32), Int64); VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1)); - VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast, {}); + VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, Cast); checkVPRecipeCastImpl(&Recipe); delete Cast; @@ -1263,7 +1263,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { SmallVector Args; Args.push_back(Op1); Args.push_back(Op2); - VPWidenRecipe Recipe(*AI, Args, VPIRMetadata(), DebugLoc()); + VPWidenRecipe Recipe(*AI, Args); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1282,7 +1282,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(Op1); Args.push_back(Op2); Args.push_back(Op3); - VPWidenSelectRecipe Recipe(*SelectI, make_range(Args.begin(), Args.end())); + VPWidenSelectRecipe Recipe(SelectI, make_range(Args.begin(), Args.end())); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1411,7 +1411,7 @@ TEST_F(VPRecipeTest, MayHaveSideEffectsAndMayReadWriteMemory) { Args.push_back(Op1); Args.push_back(Op2); Args.push_back(CalledFn); - VPWidenCallRecipe Recipe(Call, TheFn, Args); + VPWidenCallRecipe Recipe(Call, TheFn, Args, VPIRFlags(), VPIRMetadata()); EXPECT_FALSE(Recipe.mayHaveSideEffects()); EXPECT_FALSE(Recipe.mayReadFromMemory()); EXPECT_FALSE(Recipe.mayWriteToMemory()); @@ -1467,8 +1467,7 @@ TEST_F(VPRecipeTest, dumpRecipeInPlan) { VPValue *ExtVPV2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2)); Args.push_back(ExtVPV1); Args.push_back(ExtVPV2); - VPWidenRecipe *WidenR = - new VPWidenRecipe(*AI, Args, VPIRMetadata(), DebugLoc()); + VPWidenRecipe *WidenR = new VPWidenRecipe(*AI, Args); VPBB1->appendRecipe(WidenR); { From 5201a839db103b93dece983b6437051ae7772bee Mon Sep 17 00:00:00 2001 From: Florian Hahn Date: Tue, 18 Nov 2025 11:28:15 +0000 Subject: [PATCH 2/2] !fixup dyn_cast_or_null -> cast_or_null. --- llvm/lib/Transforms/Vectorize/VPlan.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 30c373de30754..de12c8b94d441 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1525,10 +1525,9 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata { ~VPWidenCastRecipe() override = default; VPWidenCastRecipe *clone() override { - return new VPWidenCastRecipe( - Opcode, getOperand(0), ResultTy, - dyn_cast_or_null(getUnderlyingValue()), *this, *this, - getDebugLoc()); + return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, + cast_or_null(getUnderlyingValue()), + *this, *this, getDebugLoc()); } VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)