diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c5026c7558b743..73ac508c389a67 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -7946,19 +7946,19 @@ void VPInterleaveRecipe::print(raw_ostream &O, const Twine &Indent, } void VPWidenCallRecipe::execute(VPTransformState &State) { - State.ILV->widenCallInstruction(Ingredient, User, State); + State.ILV->widenCallInstruction(Ingredient, *this, State); } void VPWidenSelectRecipe::execute(VPTransformState &State) { - State.ILV->widenSelectInstruction(Ingredient, User, InvariantCond, State); + State.ILV->widenSelectInstruction(Ingredient, *this, InvariantCond, State); } void VPWidenRecipe::execute(VPTransformState &State) { - State.ILV->widenInstruction(Ingredient, User, State); + State.ILV->widenInstruction(Ingredient, *this, State); } void VPWidenGEPRecipe::execute(VPTransformState &State) { - State.ILV->widenGEP(GEP, User, State.UF, State.VF, IsPtrLoopInvariant, + State.ILV->widenGEP(GEP, *this, State.UF, State.VF, IsPtrLoopInvariant, IsIndexLoopInvariant, State); } @@ -8039,7 +8039,7 @@ void VPReductionRecipe::execute(VPTransformState &State) { void VPReplicateRecipe::execute(VPTransformState &State) { if (State.Instance) { // Generate a single instance. - State.ILV->scalarizeInstruction(Ingredient, User, *State.Instance, + State.ILV->scalarizeInstruction(Ingredient, *this, *State.Instance, IsPredicated, State); // Insert scalar instance packing it into a vector. if (AlsoPack && State.VF.isVector()) { @@ -8061,7 +8061,7 @@ void VPReplicateRecipe::execute(VPTransformState &State) { unsigned EndLane = IsUniform ? 1 : State.VF.getKnownMinValue(); for (unsigned Part = 0; Part < State.UF; ++Part) for (unsigned Lane = 0; Lane < EndLane; ++Lane) - State.ILV->scalarizeInstruction(Ingredient, User, {Part, Lane}, + State.ILV->scalarizeInstruction(Ingredient, *this, {Part, Lane}, IsPredicated, State); } diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 9d1368e6c32043..416a79eacfa791 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -678,6 +678,18 @@ class VPRecipeBase : public ilist_node_with_parent { iplist::iterator eraseFromParent(); }; +inline bool VPUser::classof(const VPRecipeBase *Recipe) { + return Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPWidenCallSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPWidenSelectSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPWidenGEPSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPBlendSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPInterleaveSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPReplicateSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPBranchOnMaskSC || + Recipe->getVPRecipeID() == VPRecipeBase::VPWidenMemoryInstructionSC; +} + /// This is a concrete Recipe that models a single VPlan-level instruction. /// While as any Recipe it may generate a sequence of IR instructions when /// executed, these instructions would always form a single-def expression as @@ -780,17 +792,14 @@ class VPInstruction : public VPUser, public VPValue, public VPRecipeBase { /// VPWidenRecipe is a recipe for producing a copy of vector type its /// ingredient. This recipe covers most of the traditional vectorization cases /// where each ingredient transforms into a vectorized version of itself. -class VPWidenRecipe : public VPRecipeBase { +class VPWidenRecipe : public VPRecipeBase, public VPUser { /// Hold the instruction to be widened. Instruction &Ingredient; - /// Hold VPValues for the operands of the ingredient. - VPUser User; - public: template VPWidenRecipe(Instruction &I, iterator_range Operands) - : VPRecipeBase(VPWidenSC), Ingredient(I), User(Operands) {} + : VPRecipeBase(VPWidenSC), VPUser(Operands), Ingredient(I) {} ~VPWidenRecipe() override = default; @@ -808,17 +817,14 @@ class VPWidenRecipe : public VPRecipeBase { }; /// A recipe for widening Call instructions. -class VPWidenCallRecipe : public VPRecipeBase { +class VPWidenCallRecipe : public VPRecipeBase, public VPUser { /// Hold the call to be widened. CallInst &Ingredient; - /// Hold VPValues for the arguments of the call. - VPUser User; - public: template VPWidenCallRecipe(CallInst &I, iterator_range CallArguments) - : VPRecipeBase(VPWidenCallSC), Ingredient(I), User(CallArguments) {} + : VPRecipeBase(VPWidenCallSC), VPUser(CallArguments), Ingredient(I) {} ~VPWidenCallRecipe() override = default; @@ -836,14 +842,11 @@ class VPWidenCallRecipe : public VPRecipeBase { }; /// A recipe for widening select instructions. -class VPWidenSelectRecipe : public VPRecipeBase { +class VPWidenSelectRecipe : public VPRecipeBase, public VPUser { private: /// Hold the select to be widened. SelectInst &Ingredient; - /// Hold VPValues for the operands of the select. - VPUser User; - /// Is the condition of the select loop invariant? bool InvariantCond; @@ -851,7 +854,7 @@ class VPWidenSelectRecipe : public VPRecipeBase { template VPWidenSelectRecipe(SelectInst &I, iterator_range Operands, bool InvariantCond) - : VPRecipeBase(VPWidenSelectSC), Ingredient(I), User(Operands), + : VPRecipeBase(VPWidenSelectSC), VPUser(Operands), Ingredient(I), InvariantCond(InvariantCond) {} ~VPWidenSelectRecipe() override = default; @@ -870,20 +873,22 @@ class VPWidenSelectRecipe : public VPRecipeBase { }; /// A recipe for handling GEP instructions. -class VPWidenGEPRecipe : public VPRecipeBase { +class VPWidenGEPRecipe : public VPRecipeBase, public VPUser { GetElementPtrInst *GEP; - /// Hold VPValues for the base and indices of the GEP. - VPUser User; - bool IsPtrLoopInvariant; SmallBitVector IsIndexLoopInvariant; public: + template + VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands) + : VPRecipeBase(VPWidenGEPSC), VPUser(Operands), GEP(GEP), + IsIndexLoopInvariant(GEP->getNumIndices(), false) {} + template VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range Operands, Loop *OrigLoop) - : VPRecipeBase(VPWidenGEPSC), GEP(GEP), User(Operands), + : VPRecipeBase(VPWidenGEPSC), VPUser(Operands), GEP(GEP), IsIndexLoopInvariant(GEP->getNumIndices(), false) { IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand()); for (auto Index : enumerate(GEP->indices())) @@ -953,17 +958,15 @@ class VPWidenPHIRecipe : public VPRecipeBase { /// A recipe for vectorizing a phi-node as a sequence of mask-based select /// instructions. -class VPBlendRecipe : public VPRecipeBase { +class VPBlendRecipe : public VPRecipeBase, public VPUser { PHINode *Phi; +public: /// The blend operation is a User of the incoming values and of their /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value /// might be incoming with a full mask for which there is no VPValue. - VPUser User; - -public: VPBlendRecipe(PHINode *Phi, ArrayRef Operands) - : VPRecipeBase(VPBlendSC), Phi(Phi), User(Operands) { + : VPRecipeBase(VPBlendSC), VPUser(Operands), Phi(Phi) { assert(Operands.size() > 0 && ((Operands.size() == 1) || (Operands.size() % 2 == 0)) && "Expected either a single incoming value or a positive even number " @@ -977,17 +980,13 @@ class VPBlendRecipe : public VPRecipeBase { /// Return the number of incoming values, taking into account that a single /// incoming value has no mask. - unsigned getNumIncomingValues() const { - return (User.getNumOperands() + 1) / 2; - } + unsigned getNumIncomingValues() const { return (getNumOperands() + 1) / 2; } /// Return incoming value number \p Idx. - VPValue *getIncomingValue(unsigned Idx) const { - return User.getOperand(Idx * 2); - } + VPValue *getIncomingValue(unsigned Idx) const { return getOperand(Idx * 2); } /// Return mask number \p Idx. - VPValue *getMask(unsigned Idx) const { return User.getOperand(Idx * 2 + 1); } + VPValue *getMask(unsigned Idx) const { return getOperand(Idx * 2 + 1); } /// Generate the phi/select nodes. void execute(VPTransformState &State) override; @@ -999,16 +998,15 @@ class VPBlendRecipe : public VPRecipeBase { /// VPInterleaveRecipe is a recipe for transforming an interleave group of load /// or stores into one wide load/store and shuffles. -class VPInterleaveRecipe : public VPRecipeBase { +class VPInterleaveRecipe : public VPRecipeBase, public VPUser { const InterleaveGroup *IG; - VPUser User; public: VPInterleaveRecipe(const InterleaveGroup *IG, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPInterleaveSC), IG(IG), User({Addr}) { + : VPRecipeBase(VPInterleaveSC), VPUser({Addr}), IG(IG) { if (Mask) - User.addOperand(Mask); + addOperand(Mask); } ~VPInterleaveRecipe() override = default; @@ -1019,14 +1017,14 @@ class VPInterleaveRecipe : public VPRecipeBase { /// Return the address accessed by this recipe. VPValue *getAddr() const { - return User.getOperand(0); // Address is the 1st, mandatory operand. + return getOperand(0); // Address is the 1st, mandatory operand. } /// Return the mask used by this recipe. Note that a full mask is represented /// by a nullptr. VPValue *getMask() const { // Mask is optional and therefore the last, currently 2nd operand. - return User.getNumOperands() == 2 ? User.getOperand(1) : nullptr; + return getNumOperands() == 2 ? getOperand(1) : nullptr; } /// Generate the wide load or store, and shuffles. @@ -1080,13 +1078,10 @@ class VPReductionRecipe : public VPRecipeBase { /// copies of the original scalar type, one per lane, instead of producing a /// single copy of widened type for all lanes. If the instruction is known to be /// uniform only one copy, per lane zero, will be generated. -class VPReplicateRecipe : public VPRecipeBase { +class VPReplicateRecipe : public VPRecipeBase, public VPUser { /// The instruction being replicated. Instruction *Ingredient; - /// Hold VPValues for the operands of the ingredient. - VPUser User; - /// Indicator if only a single replica per lane is needed. bool IsUniform; @@ -1100,7 +1095,7 @@ class VPReplicateRecipe : public VPRecipeBase { template VPReplicateRecipe(Instruction *I, iterator_range Operands, bool IsUniform, bool IsPredicated = false) - : VPRecipeBase(VPReplicateSC), Ingredient(I), User(Operands), + : VPRecipeBase(VPReplicateSC), VPUser(Operands), Ingredient(I), IsUniform(IsUniform), IsPredicated(IsPredicated) { // Retain the previous behavior of predicateInstructions(), where an // insert-element of a predicated instruction got hoisted into the @@ -1130,13 +1125,11 @@ class VPReplicateRecipe : public VPRecipeBase { }; /// A recipe for generating conditional branches on the bits of a mask. -class VPBranchOnMaskRecipe : public VPRecipeBase { - VPUser User; - +class VPBranchOnMaskRecipe : public VPRecipeBase, public VPUser { public: VPBranchOnMaskRecipe(VPValue *BlockInMask) : VPRecipeBase(VPBranchOnMaskSC) { if (BlockInMask) // nullptr means all-one mask. - User.addOperand(BlockInMask); + addOperand(BlockInMask); } /// Method to support type inquiry through isa, cast, and dyn_cast. @@ -1162,9 +1155,9 @@ class VPBranchOnMaskRecipe : public VPRecipeBase { /// Return the mask used by this recipe. Note that a full mask is represented /// by a nullptr. VPValue *getMask() const { - assert(User.getNumOperands() <= 1 && "should have either 0 or 1 operands"); + assert(getNumOperands() <= 1 && "should have either 0 or 1 operands"); // Mask is optional. - return User.getNumOperands() == 1 ? User.getOperand(0) : nullptr; + return getNumOperands() == 1 ? getOperand(0) : nullptr; } }; @@ -1202,31 +1195,30 @@ class VPPredInstPHIRecipe : public VPRecipeBase { /// - For store: Address, stored value, optional mask /// TODO: We currently execute only per-part unless a specific instance is /// provided. -class VPWidenMemoryInstructionRecipe : public VPRecipeBase { +class VPWidenMemoryInstructionRecipe : public VPRecipeBase, public VPUser { Instruction &Instr; - VPUser User; void setMask(VPValue *Mask) { if (!Mask) return; - User.addOperand(Mask); + addOperand(Mask); } bool isMasked() const { - return (isa(Instr) && User.getNumOperands() == 2) || - (isa(Instr) && User.getNumOperands() == 3); + return (isa(Instr) && getNumOperands() == 2) || + (isa(Instr) && getNumOperands() == 3); } public: VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), Instr(Load), User({Addr}) { + : VPRecipeBase(VPWidenMemoryInstructionSC), VPUser({Addr}), Instr(Load) { setMask(Mask); } VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask) - : VPRecipeBase(VPWidenMemoryInstructionSC), Instr(Store), - User({Addr, StoredValue}) { + : VPRecipeBase(VPWidenMemoryInstructionSC), VPUser({Addr, StoredValue}), + Instr(Store) { setMask(Mask); } @@ -1237,21 +1229,21 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase { /// Return the address accessed by this recipe. VPValue *getAddr() const { - return User.getOperand(0); // Address is the 1st, mandatory operand. + return getOperand(0); // Address is the 1st, mandatory operand. } /// Return the mask used by this recipe. Note that a full mask is represented /// by a nullptr. VPValue *getMask() const { // Mask is optional and therefore the last operand. - return isMasked() ? User.getOperand(User.getNumOperands() - 1) : nullptr; + return isMasked() ? getOperand(getNumOperands() - 1) : nullptr; } /// Return the address accessed by this recipe. VPValue *getStoredValue() const { assert(isa(Instr) && "Stored value only available for store instructions"); - return User.getOperand(1); // Stored value is the 2nd, mandatory operand. + return getOperand(1); // Stored value is the 2nd, mandatory operand. } /// Generate the wide load/store. diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index 3274b6cf979022..50cf1285dd4b33 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -31,6 +31,7 @@ class raw_ostream; class Value; class VPSlotTracker; class VPUser; +class VPRecipeBase; // This is the base class of the VPlan Def/Use graph, used for modeling the data // flow into, within and out of the VPlan. VPValues can stand for live-ins @@ -178,6 +179,9 @@ class VPUser { const_operand_range operands() const { return const_operand_range(op_begin(), op_end()); } + + /// Method to support type inquiry through isa, cast, and dyn_cast. + static inline bool classof(const VPRecipeBase *Recipe); }; class VPlan; class VPBasicBlock; diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 73e01fade9a976..46d9899cd054c5 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -247,5 +247,152 @@ compound=true } } +TEST(VPRecipeTest, CastVPWidenRecipeToVPUser) { + LLVMContext C; + + IntegerType *Int32 = IntegerType::get(C, 32); + auto *AI = + BinaryOperator::CreateAdd(UndefValue::get(Int32), UndefValue::get(Int32)); + VPValue Op1; + VPValue Op2; + SmallVector Args; + Args.push_back(&Op1); + Args.push_back(&Op1); + VPWidenRecipe WidenR(*AI, make_range(Args.begin(), Args.end())); + EXPECT_TRUE(isa(&WidenR)); + VPRecipeBase *WidenRBase = &WidenR; + EXPECT_TRUE(isa(WidenRBase)); + delete AI; +} + +TEST(VPRecipeTest, CastVPWidenCallRecipeToVPUser) { + LLVMContext C; + + IntegerType *Int32 = IntegerType::get(C, 32); + FunctionType *FTy = FunctionType::get(Int32, false); + auto *Call = CallInst::Create(FTy, UndefValue::get(FTy)); + VPValue Op1; + VPValue Op2; + SmallVector Args; + Args.push_back(&Op1); + Args.push_back(&Op2); + VPWidenCallRecipe Recipe(*Call, make_range(Args.begin(), Args.end())); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); + delete Call; +} + +TEST(VPRecipeTest, CastVPWidenSelectRecipeToVPUser) { + LLVMContext C; + + IntegerType *Int1 = IntegerType::get(C, 1); + IntegerType *Int32 = IntegerType::get(C, 32); + auto *SelectI = SelectInst::Create( + UndefValue::get(Int1), UndefValue::get(Int32), UndefValue::get(Int32)); + VPValue Op1; + VPValue Op2; + VPValue Op3; + SmallVector Args; + Args.push_back(&Op1); + Args.push_back(&Op2); + Args.push_back(&Op3); + VPWidenSelectRecipe WidenSelectR(*SelectI, + make_range(Args.begin(), Args.end()), false); + EXPECT_TRUE(isa(&WidenSelectR)); + VPRecipeBase *BaseR = &WidenSelectR; + EXPECT_TRUE(isa(BaseR)); + delete SelectI; +} + +TEST(VPRecipeTest, CastVPWidenGEPRecipeToVPUser) { + LLVMContext C; + + IntegerType *Int32 = IntegerType::get(C, 32); + PointerType *Int32Ptr = PointerType::get(Int32, 0); + auto *GEP = GetElementPtrInst::Create(Int32, UndefValue::get(Int32Ptr), + UndefValue::get(Int32)); + VPValue Op1; + VPValue Op2; + SmallVector Args; + Args.push_back(&Op1); + Args.push_back(&Op2); + VPWidenGEPRecipe Recipe(GEP, make_range(Args.begin(), Args.end())); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); + delete GEP; +} + +TEST(VPRecipeTest, CastVPBlendRecipeToVPUser) { + LLVMContext C; + + IntegerType *Int32 = IntegerType::get(C, 32); + auto *Phi = PHINode::Create(Int32, 1); + VPValue Op1; + VPValue Op2; + SmallVector Args; + Args.push_back(&Op1); + Args.push_back(&Op2); + VPBlendRecipe Recipe(Phi, Args); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); + delete Phi; +} + +TEST(VPRecipeTest, CastVPInterleaveRecipeToVPUser) { + LLVMContext C; + + VPValue Addr; + VPValue Mask; + VPInterleaveRecipe Recipe(nullptr, &Addr, &Mask); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); +} + +TEST(VPRecipeTest, CastVPReplicateRecipeToVPUser) { + LLVMContext C; + + VPValue Op1; + VPValue Op2; + SmallVector Args; + Args.push_back(&Op1); + Args.push_back(&Op2); + + VPReplicateRecipe Recipe(nullptr, make_range(Args.begin(), Args.end()), true, + false); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); +} + +TEST(VPRecipeTest, CastVPBranchOnMaskRecipeToVPUser) { + LLVMContext C; + + VPValue Mask; + VPBranchOnMaskRecipe Recipe(&Mask); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); +} + +TEST(VPRecipeTest, CastVPWidenMemoryInstructionRecipeToVPUser) { + LLVMContext C; + + IntegerType *Int32 = IntegerType::get(C, 32); + PointerType *Int32Ptr = PointerType::get(Int32, 0); + auto *Load = + new LoadInst(Int32, UndefValue::get(Int32Ptr), "", false, Align(1)); + VPValue Addr; + VPValue Mask; + VPWidenMemoryInstructionRecipe Recipe(*Load, &Addr, &Mask); + EXPECT_TRUE(isa(&Recipe)); + VPRecipeBase *BaseR = &Recipe; + EXPECT_TRUE(isa(BaseR)); + delete Load; +} + } // namespace } // namespace llvm