Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
Original file line number Diff line number Diff line change
Expand Up @@ -152,11 +152,12 @@ class VPBuilder {
/// its underlying Instruction.
VPInstruction *createNaryOp(unsigned Opcode, ArrayRef<VPValue *> Operands,
Instruction *Inst = nullptr,
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown(),
const Twine &Name = "") {
VPInstruction *NewVPInst = tryInsertInstruction(
new VPInstruction(Opcode, Operands, {}, MD, DL, Name));
new VPInstruction(Opcode, Operands, Flags, MD, DL, Name));
NewVPInst->setUnderlyingValue(Inst);
return NewVPInst;
}
Expand Down Expand Up @@ -329,7 +330,7 @@ class VPBuilder {
else if (Opcode == Instruction::ZExt)
Flags = VPIRFlags::NonNegFlagsTy(false);
return tryInsertInstruction(
new VPWidenCastRecipe(Opcode, Op, ResultTy, Flags));
new VPWidenCastRecipe(Opcode, Op, ResultTy, nullptr, Flags));
}

VPScalarIVStepsRecipe *
Expand Down
35 changes: 20 additions & 15 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7750,7 +7750,7 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
},
Range);
if (ShouldUseVectorIntrinsic)
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI,
return new VPWidenIntrinsicRecipe(*CI, ID, Ops, CI->getType(), *VPI, *VPI,
VPI->getDebugLoc());

Function *Variant = nullptr;
Expand Down Expand Up @@ -7804,7 +7804,8 @@ VPSingleDefRecipe *VPRecipeBuilder::tryToWidenCall(VPInstruction *VPI,
}

Ops.push_back(VPI->getOperand(VPI->getNumOperands() - 1));
return new VPWidenCallRecipe(CI, Variant, Ops, VPI->getDebugLoc());
return new VPWidenCallRecipe(CI, Variant, Ops, *VPI, *VPI,
VPI->getDebugLoc());
}

return nullptr;
Expand Down Expand Up @@ -7842,7 +7843,7 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
auto *SafeRHS =
Builder.createSelect(Mask, Ops[1], One, VPI->getDebugLoc());
Ops[1] = SafeRHS;
return new VPWidenRecipe(*I, Ops, *VPI, VPI->getDebugLoc());
return new VPWidenRecipe(*I, Ops, *VPI, *VPI, VPI->getDebugLoc());
}
[[fallthrough]];
}
Expand Down Expand Up @@ -7888,15 +7889,15 @@ VPWidenRecipe *VPRecipeBuilder::tryToWiden(VPInstruction *VPI) {
// For other binops, the legacy cost model only checks the second operand.
NewOps[1] = GetConstantViaSCEV(NewOps[1]);
}
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
}
case Instruction::ExtractValue: {
SmallVector<VPValue *> NewOps(VPI->operands());
auto *EVI = cast<ExtractValueInst>(I);
assert(EVI->getNumIndices() == 1 && "Expected one extractvalue index");
unsigned Idx = EVI->getIndices()[0];
NewOps.push_back(Plan.getConstantInt(32, Idx));
return new VPWidenRecipe(*I, NewOps, *VPI, VPI->getDebugLoc());
return new VPWidenRecipe(*I, NewOps, *VPI, *VPI, VPI->getDebugLoc());
}
};
}
Expand Down Expand Up @@ -7981,7 +7982,8 @@ VPReplicateRecipe *VPRecipeBuilder::handleReplication(VPInstruction *VPI,
(Range.Start.isScalable() && isa<IntrinsicInst>(I))) &&
"Should not predicate a uniform recipe");
auto *Recipe =
new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI);
new VPReplicateRecipe(I, VPI->operands(), IsUniform, BlockInMask, *VPI,
*VPI, VPI->getDebugLoc());
return Recipe;
}

Expand Down Expand Up @@ -8231,17 +8233,19 @@ VPRecipeBase *VPRecipeBuilder::tryToCreateWidenRecipe(VPSingleDefRecipe *R,
return nullptr;

if (VPI->getOpcode() == Instruction::GetElementPtr)
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands());
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(Instr), R->operands(),
*VPI, VPI->getDebugLoc());

if (VPI->getOpcode() == Instruction::Select)
return new VPWidenSelectRecipe(*cast<SelectInst>(Instr), R->operands(),
*VPI);
return new VPWidenSelectRecipe(cast<SelectInst>(Instr), R->operands(), *VPI,
*VPI, VPI->getDebugLoc());

if (Instruction::isCast(VPI->getOpcode())) {
auto *CastR = cast<VPInstructionWithType>(R);
auto *CI = cast<CastInst>(Instr);
auto *CastR = cast<VPInstructionWithType>(VPI);
return new VPWidenCastRecipe(CI->getOpcode(), VPI->getOperand(0),
CastR->getResultType(), *CI, *VPI);
CastR->getResultType(), CI, *VPI, *VPI,
VPI->getDebugLoc());
}

return tryToWiden(VPI);
Expand Down Expand Up @@ -8269,8 +8273,8 @@ VPRecipeBuilder::tryToCreatePartialReduction(VPInstruction *Reduction,
SmallVector<VPValue *, 2> Ops;
Ops.push_back(Plan.getOrAddLiveIn(Zero));
Ops.push_back(BinOp);
BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRMetadata(),
ReductionI->getDebugLoc());
BinOp = new VPWidenRecipe(*ReductionI, Ops, VPIRFlags(*ReductionI),
VPIRMetadata(), ReductionI->getDebugLoc());
Builder.insert(BinOp->getDefiningRecipe());
ReductionOpcode = Instruction::Add;
}
Expand Down Expand Up @@ -8454,9 +8458,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
Legal->isInvariantAddressOfReduction(SI->getPointerOperand())) {
// Only create recipe for the final invariant store of the reduction.
if (Legal->isInvariantStoreOfReduction(SI)) {
auto *VPI = cast<VPInstruction>(SingleDef);
auto *Recipe = new VPReplicateRecipe(
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/,
*cast<VPInstruction>(SingleDef));
SI, R.operands(), true /* IsUniform */, nullptr /*Mask*/, *VPI,
*VPI, VPI->getDebugLoc());
Recipe->insertBefore(*MiddleVPBB, MBIP);
}
R.eraseFromParent();
Expand Down
92 changes: 46 additions & 46 deletions llvm/lib/Transforms/Vectorize/VPlan.h
Original file line number Diff line number Diff line change
Expand Up @@ -882,14 +882,6 @@ class VPIRFlags {
/// A pure-virtual common base class for recipes defining a single VPValue and
/// using IR flags.
struct VPRecipeWithIRFlags : public VPSingleDefRecipe, public VPIRFlags {
VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
DebugLoc DL = DebugLoc::getUnknown())
: VPSingleDefRecipe(SC, Operands, DL), VPIRFlags() {}

VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
Instruction &I)
: VPSingleDefRecipe(SC, Operands, &I, I.getDebugLoc()), VPIRFlags(I) {}

VPRecipeWithIRFlags(const unsigned char SC, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags,
DebugLoc DL = DebugLoc::getUnknown())
Expand Down Expand Up @@ -1474,9 +1466,12 @@ class LLVM_ABI_FOR_TEST VPWidenRecipe : public VPRecipeWithIRFlags,
VPIRMetadata(Metadata), Opcode(Opcode) {}

VPWidenRecipe(Instruction &I, ArrayRef<VPValue *> Operands,
const VPIRMetadata &Metadata, DebugLoc DL)
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, I),
VPIRMetadata(Metadata), Opcode(I.getOpcode()) {}
const VPIRFlags &Flags = {}, const VPIRMetadata &Metadata = {},
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenSC, Operands, Flags, DL),
VPIRMetadata(Metadata), Opcode(I.getOpcode()) {
setUnderlyingValue(&I);
}

~VPWidenRecipe() override = default;

Expand Down Expand Up @@ -1517,30 +1512,22 @@ class VPWidenCastRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {

public:
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
CastInst &UI, const VPIRMetadata &Metadata)
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, UI),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(UI.getOpcode() == Opcode &&
"opcode of underlying cast doesn't match");
}
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy,
const VPIRFlags &Flags = {},
CastInst *CI = nullptr, const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenCastSC, Op, Flags, DL),
VPIRMetadata(Metadata), Opcode(Opcode), ResultTy(ResultTy) {
assert(flagsValidForOpcode(Opcode) &&
"Set flags not supported for the provided opcode");
setUnderlyingValue(CI);
}

~VPWidenCastRecipe() override = default;

VPWidenCastRecipe *clone() override {
auto *New = new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy, *this,
*this, getDebugLoc());
if (auto *UV = getUnderlyingValue())
New->setUnderlyingValue(UV);
return New;
return new VPWidenCastRecipe(Opcode, getOperand(0), ResultTy,
cast_or_null<CastInst>(getUnderlyingValue()),
*this, *this, getDebugLoc());
}

VP_CLASSOF_IMPL(VPDef::VPWidenCastSC)
Expand Down Expand Up @@ -1585,13 +1572,17 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
public:
VPWidenIntrinsicRecipe(CallInst &CI, Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
const VPIRFlags &Flags = {},
const VPIRMetadata &MD = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, CI),
: VPRecipeWithIRFlags(VPDef::VPWidenIntrinsicSC, CallArguments, Flags,
DL),
VPIRMetadata(MD), VectorIntrinsicID(VectorIntrinsicID), ResultTy(Ty),
MayReadFromMemory(CI.mayReadFromMemory()),
MayWriteToMemory(CI.mayWriteToMemory()),
MayHaveSideEffects(CI.mayHaveSideEffects()) {}
MayHaveSideEffects(CI.mayHaveSideEffects()) {
setUnderlyingValue(&CI);
}

VPWidenIntrinsicRecipe(Intrinsic::ID VectorIntrinsicID,
ArrayRef<VPValue *> CallArguments, Type *Ty,
Expand All @@ -1617,7 +1608,7 @@ class VPWidenIntrinsicRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
VPWidenIntrinsicRecipe *clone() override {
if (Value *CI = getUnderlyingValue())
return new VPWidenIntrinsicRecipe(*cast<CallInst>(CI), VectorIntrinsicID,
operands(), ResultTy, *this,
operands(), ResultTy, *this, *this,
getDebugLoc());
return new VPWidenIntrinsicRecipe(VectorIntrinsicID, operands(), ResultTy,
*this, *this, getDebugLoc());
Expand Down Expand Up @@ -1671,10 +1662,11 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,
public:
VPWidenCallRecipe(Value *UV, Function *Variant,
ArrayRef<VPValue *> CallArguments,
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments,
*cast<Instruction>(UV)),
VPIRMetadata(*cast<Instruction>(UV)), Variant(Variant) {
const VPIRFlags &Flags = {},
const VPIRMetadata &Metadata = {}, DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenCallSC, CallArguments, Flags, DL),
VPIRMetadata(Metadata), Variant(Variant) {
setUnderlyingValue(UV);
assert(
isa<Function>(getOperand(getNumOperands() - 1)->getLiveInIRValue()) &&
"last operand must be the called function");
Expand All @@ -1684,7 +1676,7 @@ class LLVM_ABI_FOR_TEST VPWidenCallRecipe : public VPRecipeWithIRFlags,

VPWidenCallRecipe *clone() override {
return new VPWidenCallRecipe(getUnderlyingValue(), Variant, operands(),
getDebugLoc());
*this, *this, getDebugLoc());
}

VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)
Expand Down Expand Up @@ -1761,16 +1753,19 @@ class VPHistogramRecipe : public VPRecipeBase {
/// instruction.
struct LLVM_ABI_FOR_TEST VPWidenSelectRecipe : public VPRecipeWithIRFlags,
public VPIRMetadata {
VPWidenSelectRecipe(SelectInst &I, ArrayRef<VPValue *> Operands,
const VPIRMetadata &MD = {})
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, I),
VPIRMetadata(MD) {}
VPWidenSelectRecipe(SelectInst *SI, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {}, const VPIRMetadata &MD = {},
DebugLoc DL = {})
: VPRecipeWithIRFlags(VPDef::VPWidenSelectSC, Operands, Flags, DL),
VPIRMetadata(MD) {
setUnderlyingValue(SI);
}

~VPWidenSelectRecipe() override = default;

VPWidenSelectRecipe *clone() override {
return new VPWidenSelectRecipe(*cast<SelectInst>(getUnderlyingInstr()),
operands(), *this);
return new VPWidenSelectRecipe(cast<SelectInst>(getUnderlyingInstr()),
operands(), *this, *this, getDebugLoc());
}

VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)
Expand Down Expand Up @@ -1822,9 +1817,12 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {
}

public:
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands)
: VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, *GEP),
VPWidenGEPRecipe(GetElementPtrInst *GEP, ArrayRef<VPValue *> Operands,
const VPIRFlags &Flags = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPWidenGEPSC, Operands, Flags, DL),
SourceElementTy(GEP->getSourceElementType()) {
setUnderlyingValue(GEP);
SmallVector<std::pair<unsigned, MDNode *>> Metadata;
(void)Metadata;
getMetadataToPropagate(GEP, Metadata);
Expand All @@ -1835,7 +1833,7 @@ class LLVM_ABI_FOR_TEST VPWidenGEPRecipe : public VPRecipeWithIRFlags {

VPWidenGEPRecipe *clone() override {
return new VPWidenGEPRecipe(cast<GetElementPtrInst>(getUnderlyingInstr()),
operands());
operands(), *this, getDebugLoc());
}

VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)
Expand Down Expand Up @@ -2929,20 +2927,22 @@ class LLVM_ABI_FOR_TEST VPReplicateRecipe : public VPRecipeWithIRFlags,
public:
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
bool IsSingleScalar, VPValue *Mask = nullptr,
VPIRMetadata Metadata = {})
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
const VPIRFlags &Flags = {}, VPIRMetadata Metadata = {},
DebugLoc DL = DebugLoc::getUnknown())
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, Flags, DL),
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
IsPredicated(Mask) {
setUnderlyingValue(I);
if (Mask)
addOperand(Mask);
}

~VPReplicateRecipe() override = default;

VPReplicateRecipe *clone() override {
auto *Copy =
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsSingleScalar,
isPredicated() ? getMask() : nullptr, *this);
auto *Copy = new VPReplicateRecipe(
getUnderlyingInstr(), operands(), IsSingleScalar,
isPredicated() ? getMask() : nullptr, *this, *this, getDebugLoc());
Copy->transferFlags(*this);
return Copy;
}
Expand Down
11 changes: 6 additions & 5 deletions llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
// recipes.
if (Br->isConditional()) {
VPValue *Cond = getOrCreateVPOperand(Br->getCondition());
VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst,
VPIRBuilder.createNaryOp(VPInstruction::BranchOnCond, {Cond}, Inst, {},
VPIRMetadata(*Inst), Inst->getDebugLoc());
}

Expand All @@ -205,7 +205,7 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
SmallVector<VPValue *> Ops = {getOrCreateVPOperand(SI->getCondition())};
for (auto Case : SI->cases())
Ops.push_back(getOrCreateVPOperand(Case.getCaseValue()));
VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst,
VPIRBuilder.createNaryOp(Instruction::Switch, Ops, Inst, {},
VPIRMetadata(*Inst), Inst->getDebugLoc());
continue;
}
Expand Down Expand Up @@ -255,13 +255,14 @@ void PlainCFGBuilder::createVPInstructionsForVPBB(VPBasicBlock *VPBB,
if (auto *CI = dyn_cast<CastInst>(Inst)) {
NewR = VPIRBuilder.createScalarCast(CI->getOpcode(), VPOperands[0],
CI->getType(), CI->getDebugLoc(),
{}, MD);
VPIRFlags(*CI), MD);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we're propagating more flags that weren't previously do we need to drop NFC from the PR title?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be NFC w.r.t. to the generated IR, but it impacts the debug output. I removed NFC.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh I see that e.g. VPInstructionWithType::execute doesn't set the IR flags etc. I presume we should eventually do that in a separate PR?

NewR->setUnderlyingValue(CI);
} else {
// Build VPInstruction for any arbitrary Instruction without specific
// representation in VPlan.
NewR = VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst, MD,
Inst->getDebugLoc());
NewR =
VPIRBuilder.createNaryOp(Inst->getOpcode(), VPOperands, Inst,
VPIRFlags(*Inst), MD, Inst->getDebugLoc());
}
}

Expand Down
16 changes: 9 additions & 7 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2056,24 +2056,26 @@ bool VPIRFlags::flagsValidForOpcode(unsigned Opcode) const {
switch (OpType) {
case OperationType::OverflowingBinOp:
return Opcode == Instruction::Add || Opcode == Instruction::Sub ||
Opcode == Instruction::Mul ||
Opcode == Instruction::Mul || Opcode == Instruction::Shl ||
Opcode == VPInstruction::VPInstruction::CanonicalIVIncrementForPart;
case OperationType::Trunc:
return Opcode == Instruction::Trunc;
case OperationType::DisjointOp:
return Opcode == Instruction::Or;
case OperationType::PossiblyExactOp:
return Opcode == Instruction::AShr;
return Opcode == Instruction::AShr || Opcode == Instruction::LShr ||
Opcode == Instruction::UDiv || Opcode == Instruction::SDiv;
case OperationType::GEPOp:
return Opcode == Instruction::GetElementPtr ||
Opcode == VPInstruction::PtrAdd ||
Opcode == VPInstruction::WidePtrAdd;
case OperationType::FPMathOp:
return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
Opcode == Instruction::FSub || Opcode == Instruction::FNeg ||
Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
Opcode == Instruction::FPExt || Opcode == Instruction::FPTrunc ||
Opcode == Instruction::FCmp || Opcode == Instruction::Select ||
return Opcode == Instruction::Call || Opcode == Instruction::FAdd ||
Opcode == Instruction::FMul || Opcode == Instruction::FSub ||
Opcode == Instruction::FNeg || Opcode == Instruction::FDiv ||
Opcode == Instruction::FRem || Opcode == Instruction::FPExt ||
Opcode == Instruction::FPTrunc || Opcode == Instruction::FCmp ||
Opcode == Instruction::Select ||
Opcode == VPInstruction::WideIVStep ||
Opcode == VPInstruction::ReductionStartVector ||
Opcode == VPInstruction::ComputeReductionResult;
Expand Down
Loading
Loading