Skip to content

Commit

Permalink
[VPlan] Remove duplicated VPValue IDs (NFCI).
Browse files Browse the repository at this point in the history 
At the moment, both VPValue and VPDef have an ID used when casting via
classof. This duplication is cumbersome, because it requires adding IDs
for new recipes twice and also requires setting them twice. In a few
cases, there's only a VPDef ID and no VPValue ID, which can cause same
confusion.

To simplify things, remove the VPValue IDs for different recipes.
Instead, only retain the generic VPValue ID (= used VPValues without a
corresponding defining recipe) and VPVRecipe for VPValues that are
defined by recipes that inherit from VPValue.

Reviewed By: Ayal

Differential Revision: https://reviews.llvm.org/D140848
  • Loading branch information
@fhahn
fhahn committed Jan 17, 2023
1 parent 4adf89f commit d47bdae
Show file tree
Hide file tree
Showing 2 changed files with 65 additions and 103 deletions.
128 changes: 55 additions & 73 deletions llvm/lib/Transforms/Vectorize/VPlan.h
View file
Expand Up @@ -756,12 +756,13 @@ class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
};

// Helper macro to define common classof implementations for recipes.
#define VP_CLASSOF_IMPL(VPDefID, VPValueID) \
#define VP_CLASSOF_IMPL(VPDefID) \
static inline bool classof(const VPDef *D) { \
return D->getVPDefID() == VPDefID; \
} \
static inline bool classof(const VPValue *V) { \
return V->getVPValueID() == VPValueID; \
auto *R = V->getDefiningRecipe(); \
return R && R->getVPDefID() == VPDefID; \
} \
static inline bool classof(const VPUser *U) { \
auto *R = dyn_cast<VPRecipeBase>(U); \
Expand Down Expand Up @@ -818,15 +819,14 @@ class VPInstruction : public VPRecipeBase, public VPValue {
public:
VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL,
const Twine &Name = "")
: VPRecipeBase(VPDef::VPInstructionSC, Operands),
VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode),
DL(DL), Name(Name.str()) {}
: VPRecipeBase(VPDef::VPInstructionSC, Operands), VPValue(this),
Opcode(Opcode), DL(DL), Name(Name.str()) {}

VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
DebugLoc DL = {}, const Twine &Name = "")
: VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL, Name) {}

VP_CLASSOF_IMPL(VPDef::VPInstructionSC, VPValue::VPVInstructionSC)
VP_CLASSOF_IMPL(VPDef::VPInstructionSC)

VPInstruction *clone() const {
SmallVector<VPValue *, 2> Operands(operands());
Expand Down Expand Up @@ -910,12 +910,11 @@ class VPWidenRecipe : public VPRecipeBase, public VPValue {
public:
template <typename IterT>
VPWidenRecipe(Instruction &I, iterator_range<IterT> Operands)
: VPRecipeBase(VPDef::VPWidenSC, Operands),
VPValue(VPValue::VPVWidenSC, &I, this) {}
: VPRecipeBase(VPDef::VPWidenSC, Operands), VPValue(this, &I) {}

~VPWidenRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenSC, VPValue::VPVWidenSC)
VP_CLASSOF_IMPL(VPDef::VPWidenSC)

/// Produce widened copies of all Ingredients.
void execute(VPTransformState &State) override;
Expand All @@ -937,13 +936,12 @@ class VPWidenCallRecipe : public VPRecipeBase, public VPValue {
template <typename IterT>
VPWidenCallRecipe(CallInst &I, iterator_range<IterT> CallArguments,
Intrinsic::ID VectorIntrinsicID)
: VPRecipeBase(VPDef::VPWidenCallSC, CallArguments),
VPValue(VPValue::VPVWidenCallSC, &I, this),
: VPRecipeBase(VPDef::VPWidenCallSC, CallArguments), VPValue(this, &I),
VectorIntrinsicID(VectorIntrinsicID) {}

~VPWidenCallRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenCallSC, VPValue::VPVWidenCallSC)
VP_CLASSOF_IMPL(VPDef::VPWidenCallSC)

/// Produce a widened version of the call instruction.
void execute(VPTransformState &State) override;
Expand All @@ -965,13 +963,12 @@ class VPWidenSelectRecipe : public VPRecipeBase, public VPValue {
template <typename IterT>
VPWidenSelectRecipe(SelectInst &I, iterator_range<IterT> Operands,
bool InvariantCond)
: VPRecipeBase(VPDef::VPWidenSelectSC, Operands),
VPValue(VPValue::VPVWidenSelectSC, &I, this),
: VPRecipeBase(VPDef::VPWidenSelectSC, Operands), VPValue(this, &I),
InvariantCond(InvariantCond) {}

~VPWidenSelectRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC, VPValue::VPVWidenSelectSC)
VP_CLASSOF_IMPL(VPDef::VPWidenSelectSC)

/// Produce a widened version of the select instruction.
void execute(VPTransformState &State) override;
Expand All @@ -991,15 +988,13 @@ class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
public:
template <typename IterT>
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range<IterT> Operands)
: VPRecipeBase(VPDef::VPWidenGEPSC, Operands),
VPValue(VPWidenGEPSC, GEP, this),
: VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP),
IsIndexLoopInvariant(GEP->getNumIndices(), false) {}

template <typename IterT>
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range<IterT> Operands,
Loop *OrigLoop)
: VPRecipeBase(VPDef::VPWidenGEPSC, Operands),
VPValue(VPValue::VPVWidenGEPSC, GEP, this),
: VPRecipeBase(VPDef::VPWidenGEPSC, Operands), VPValue(this, GEP),
IsIndexLoopInvariant(GEP->getNumIndices(), false) {
IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand());
for (auto Index : enumerate(GEP->indices()))
Expand All @@ -1008,7 +1003,7 @@ class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
}
~VPWidenGEPRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC, VPValue::VPVWidenGEPSC)
VP_CLASSOF_IMPL(VPDef::VPWidenGEPSC)

/// Generate the gep nodes.
void execute(VPTransformState &State) override;
Expand All @@ -1032,20 +1027,19 @@ class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPValue {
const InductionDescriptor &IndDesc,
bool NeedsVectorIV)
: VPRecipeBase(VPDef::VPWidenIntOrFpInductionSC, {Start, Step}),
VPValue(IV, this), IV(IV), IndDesc(IndDesc),
VPValue(this, IV), IV(IV), IndDesc(IndDesc),
NeedsVectorIV(NeedsVectorIV) {}

VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step,
const InductionDescriptor &IndDesc,
TruncInst *Trunc, bool NeedsVectorIV)
: VPRecipeBase(VPDef::VPWidenIntOrFpInductionSC, {Start, Step}),
VPValue(Trunc, this), IV(IV), IndDesc(IndDesc),
VPValue(this, Trunc), IV(IV), IndDesc(IndDesc),
NeedsVectorIV(NeedsVectorIV) {}

~VPWidenIntOrFpInductionRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC,
VPValue::VPVWidenIntOrFpInductionSC)
VP_CLASSOF_IMPL(VPDef::VPWidenIntOrFpInductionSC)

/// Generate the vectorized and scalarized versions of the phi node as
/// needed by their users.
Expand Down Expand Up @@ -1118,9 +1112,9 @@ class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPValue {
/// per-lane based on the canonical induction.
class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue {
protected:
VPHeaderPHIRecipe(unsigned char VPVID, unsigned char VPDefID, PHINode *Phi,
VPHeaderPHIRecipe(unsigned char VPDefID, PHINode *Phi,
VPValue *Start = nullptr)
: VPRecipeBase(VPDefID, {}), VPValue(VPVID, Phi, this) {
: VPRecipeBase(VPDefID, {}), VPValue(this, Phi) {
if (Start)
addOperand(Start);
}
Expand All @@ -1134,8 +1128,9 @@ class VPHeaderPHIRecipe : public VPRecipeBase, public VPValue {
B->getVPDefID() <= VPDef::VPLastPHISC;
}
static inline bool classof(const VPValue *V) {
return V->getVPValueID() >= VPValue::VPVFirstHeaderPHISC &&
V->getVPValueID() <= VPValue::VPVLastPHISC;
auto *B = V->getDefiningRecipe();
return B && B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
B->getVPDefID() <= VPRecipeBase::VPLastPHISC;
}

/// Generate the phi nodes.
Expand Down Expand Up @@ -1181,8 +1176,7 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe {
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step,
const InductionDescriptor &IndDesc,
bool IsScalarAfterVectorization)
: VPHeaderPHIRecipe(VPVWidenPointerInductionSC,
VPDef::VPWidenPointerInductionSC, Phi),
: VPHeaderPHIRecipe(VPDef::VPWidenPointerInductionSC, Phi),
IndDesc(IndDesc),
IsScalarAfterVectorization(IsScalarAfterVectorization) {
addOperand(Start);
Expand All @@ -1191,8 +1185,7 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe {

~VPWidenPointerInductionRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC,
VPValue::VPVWidenPointerInductionSC)
VP_CLASSOF_IMPL(VPDef::VPWidenPointerInductionSC)

/// Generate vector values for the pointer induction.
void execute(VPTransformState &State) override;
Expand Down Expand Up @@ -1220,14 +1213,14 @@ class VPWidenPHIRecipe : public VPHeaderPHIRecipe {
public:
/// Create a new VPWidenPHIRecipe for \p Phi with start value \p Start.
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start = nullptr)
: VPHeaderPHIRecipe(VPVWidenPHISC, VPDef::VPWidenPHISC, Phi) {
: VPHeaderPHIRecipe(VPDef::VPWidenPHISC, Phi) {
if (Start)
addOperand(Start);
}

~VPWidenPHIRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenPHISC, VPValue::VPVWidenPHISC)
VP_CLASSOF_IMPL(VPDef::VPWidenPHISC)

/// Generate the phi/select nodes.
void execute(VPTransformState &State) override;
Expand Down Expand Up @@ -1256,11 +1249,9 @@ class VPWidenPHIRecipe : public VPHeaderPHIRecipe {
/// second operand.
struct VPFirstOrderRecurrencePHIRecipe : public VPHeaderPHIRecipe {
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
: VPHeaderPHIRecipe(VPVFirstOrderRecurrencePHISC,
VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {}
: VPHeaderPHIRecipe(VPDef::VPFirstOrderRecurrencePHISC, Phi, &Start) {}

VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC,
VPValue::VPVFirstOrderRecurrencePHISC)
VP_CLASSOF_IMPL(VPDef::VPFirstOrderRecurrencePHISC)

static inline bool classof(const VPHeaderPHIRecipe *R) {
return R->getVPDefID() == VPDef::VPFirstOrderRecurrencePHISC;
Expand Down Expand Up @@ -1294,15 +1285,14 @@ class VPReductionPHIRecipe : public VPHeaderPHIRecipe {
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
VPValue &Start, bool IsInLoop = false,
bool IsOrdered = false)
: VPHeaderPHIRecipe(VPVReductionPHISC, VPDef::VPReductionPHISC, Phi,
&Start),
: VPHeaderPHIRecipe(VPDef::VPReductionPHISC, Phi, &Start),
RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
assert((!IsOrdered || IsInLoop) && "IsOrdered requires IsInLoop");
}

~VPReductionPHIRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPReductionPHISC, VPValue::VPVReductionPHISC)
VP_CLASSOF_IMPL(VPDef::VPReductionPHISC)

static inline bool classof(const VPHeaderPHIRecipe *R) {
return R->getVPDefID() == VPDef::VPReductionPHISC;
Expand Down Expand Up @@ -1338,15 +1328,14 @@ class VPBlendRecipe : public VPRecipeBase, public VPValue {
/// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
/// might be incoming with a full mask for which there is no VPValue.
VPBlendRecipe(PHINode *Phi, ArrayRef<VPValue *> Operands)
: VPRecipeBase(VPDef::VPBlendSC, Operands),
VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) {
: VPRecipeBase(VPDef::VPBlendSC, Operands), VPValue(this, Phi), Phi(Phi) {
assert(Operands.size() > 0 &&
((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
"Expected either a single incoming value or a positive even number "
"of operands");
}

VP_CLASSOF_IMPL(VPDef::VPBlendSC, VPValue::VPVBlendSC)
VP_CLASSOF_IMPL(VPDef::VPBlendSC)

/// Return the number of incoming values, taking into account that a single
/// incoming value has no mask.
Expand Down Expand Up @@ -1470,15 +1459,15 @@ class VPReductionRecipe : public VPRecipeBase, public VPValue {
VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I,
VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
const TargetTransformInfo *TTI)
: VPRecipeBase(VPDef::VPReductionSC, {ChainOp, VecOp}),
VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
: VPRecipeBase(VPDef::VPReductionSC, {ChainOp, VecOp}), VPValue(this, I),
RdxDesc(R), TTI(TTI) {
if (CondOp)
addOperand(CondOp);
}

~VPReductionRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPReductionSC, VPValue::VPVReductionSC)
VP_CLASSOF_IMPL(VPDef::VPReductionSC)

/// Generate the reduction in the loop
void execute(VPTransformState &State) override;
Expand Down Expand Up @@ -1517,9 +1506,8 @@ class VPReplicateRecipe : public VPRecipeBase, public VPValue {
template <typename IterT>
VPReplicateRecipe(Instruction *I, iterator_range<IterT> Operands,
bool IsUniform, bool IsPredicated = false)
: VPRecipeBase(VPDef::VPReplicateSC, Operands),
VPValue(VPVReplicateSC, I, this), IsUniform(IsUniform),
IsPredicated(IsPredicated) {
: VPRecipeBase(VPDef::VPReplicateSC, Operands), VPValue(this, I),
IsUniform(IsUniform), IsPredicated(IsPredicated) {
// Retain the previous behavior of predicateInstructions(), where an
// insert-element of a predicated instruction got hoisted into the
// predicated basic block iff it was its only user. This is achieved by
Expand All @@ -1530,7 +1518,7 @@ class VPReplicateRecipe : public VPRecipeBase, public VPValue {

~VPReplicateRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPReplicateSC, VPValue::VPVReplicateSC)
VP_CLASSOF_IMPL(VPDef::VPReplicateSC)

/// Generate replicas of the desired Ingredient. Replicas will be generated
/// for all parts and lanes unless a specific part and lane are specified in
Expand Down Expand Up @@ -1622,11 +1610,10 @@ class VPPredInstPHIRecipe : public VPRecipeBase, public VPValue {
/// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
/// nodes after merging back from a Branch-on-Mask.
VPPredInstPHIRecipe(VPValue *PredV)
: VPRecipeBase(VPDef::VPPredInstPHISC, PredV),
VPValue(VPValue::VPVPredInstPHI, nullptr, this) {}
: VPRecipeBase(VPDef::VPPredInstPHISC, PredV), VPValue(this) {}
~VPPredInstPHIRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC, VPValue::VPVPredInstPHI)
VP_CLASSOF_IMPL(VPDef::VPPredInstPHISC)

/// Generates phi nodes for live-outs as needed to retain SSA form.
void execute(VPTransformState &State) override;
Expand Down Expand Up @@ -1676,7 +1663,7 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase {
: VPRecipeBase(VPDef::VPWidenMemoryInstructionSC, {Addr}),
Ingredient(Load), Consecutive(Consecutive), Reverse(Reverse) {
assert((Consecutive || !Reverse) && "Reverse implies consecutive");
new VPValue(VPValue::VPVMemoryInstructionSC, &Load, this);
new VPValue(this, &Load);
setMask(Mask);
}

Expand All @@ -1689,8 +1676,7 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase {
setMask(Mask);
}

VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC,
VPValue::VPVMemoryInstructionSC)
VP_CLASSOF_IMPL(VPDef::VPWidenMemoryInstructionSC)

/// Return the address accessed by this recipe.
VPValue *getAddr() const {
Expand Down Expand Up @@ -1751,8 +1737,8 @@ class VPExpandSCEVRecipe : public VPRecipeBase, public VPValue {

public:
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
: VPRecipeBase(VPDef::VPExpandSCEVSC, {}), VPValue(nullptr, this),
Expr(Expr), SE(SE) {}
: VPRecipeBase(VPDef::VPExpandSCEVSC, {}), VPValue(this), Expr(Expr),
SE(SE) {}

~VPExpandSCEVRecipe() override = default;

Expand Down Expand Up @@ -1782,13 +1768,11 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {

public:
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
: VPHeaderPHIRecipe(VPValue::VPVCanonicalIVPHISC,
VPDef::VPCanonicalIVPHISC, nullptr, StartV),
DL(DL) {}
: VPHeaderPHIRecipe(VPDef::VPCanonicalIVPHISC, nullptr, StartV), DL(DL) {}

~VPCanonicalIVPHIRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC, VPValue::VPVCanonicalIVPHISC)
VP_CLASSOF_IMPL(VPDef::VPCanonicalIVPHISC)

static inline bool classof(const VPHeaderPHIRecipe *D) {
return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
Expand Down Expand Up @@ -1829,13 +1813,12 @@ class VPActiveLaneMaskPHIRecipe : public VPHeaderPHIRecipe {

public:
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
: VPHeaderPHIRecipe(VPValue::VPVActiveLaneMaskPHISC,
VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask),
: VPHeaderPHIRecipe(VPDef::VPActiveLaneMaskPHISC, nullptr, StartMask),
DL(DL) {}

~VPActiveLaneMaskPHIRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC, VPValue::VPVActiveLaneMaskPHISC)
VP_CLASSOF_IMPL(VPDef::VPActiveLaneMaskPHISC)

static inline bool classof(const VPHeaderPHIRecipe *D) {
return D->getVPDefID() == VPDef::VPActiveLaneMaskPHISC;
Expand All @@ -1856,11 +1839,11 @@ class VPWidenCanonicalIVRecipe : public VPRecipeBase, public VPValue {
public:
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
: VPRecipeBase(VPDef::VPWidenCanonicalIVSC, {CanonicalIV}),
VPValue(VPValue::VPVWidenCanonicalIVSC, nullptr, this) {}
VPValue(this) {}

~VPWidenCanonicalIVRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC, VPValue::VPVWidenCanonicalIVSC)
VP_CLASSOF_IMPL(VPDef::VPWidenCanonicalIVSC)

/// Generate a canonical vector induction variable of the vector loop, with
/// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
Expand Down Expand Up @@ -1896,12 +1879,11 @@ class VPDerivedIVRecipe : public VPRecipeBase, public VPValue {
VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step,
Type *ResultTy)
: VPRecipeBase(VPDef::VPDerivedIVSC, {Start, CanonicalIV, Step}),
VPValue(VPVDerivedIVSC, nullptr, this), ResultTy(ResultTy),
IndDesc(IndDesc) {}
VPValue(this), ResultTy(ResultTy), IndDesc(IndDesc) {}

~VPDerivedIVRecipe() override = default;

VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC, VPValue::VPVInstructionSC)
VP_CLASSOF_IMPL(VPDef::VPDerivedIVSC)

/// Generate the transformed value of the induction at offset StartValue (1.
/// operand) + IV (2. operand) * StepValue (3, operand).
Expand Down Expand Up @@ -1933,8 +1915,8 @@ class VPScalarIVStepsRecipe : public VPRecipeBase, public VPValue {
public:
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV,
VPValue *Step)
: VPRecipeBase(VPDef::VPScalarIVStepsSC, {IV, Step}),
VPValue(nullptr, this), IndDesc(IndDesc) {}
: VPRecipeBase(VPDef::VPScalarIVStepsSC, {IV, Step}), VPValue(this),
IndDesc(IndDesc) {}

~VPScalarIVStepsRecipe() override = default;

Expand Down

0 comments on commit d47bdae

Please sign in to comment.