diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index a88ddf217da9b..fc29ab0c84093 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -3876,6 +3876,75 @@ template <> struct CastInfo : CastInfoVPPhiAccessors {}; +/// Casting from (const) VPRecipeBase -> (const) VPIRMetadata is supported for +/// all recipe types implementing VPIRMetadata. Used by isa<> & co. +namespace detail { +template +static inline auto castToVPIRMetadata(RecipeBasePtrTy R) -> DstTy { + switch (R->getVPDefID()) { + case VPDef::VPInstructionSC: + return cast(R); + case VPDef::VPWidenSC: + return cast(R); + case VPDef::VPWidenCastSC: + return cast(R); + case VPDef::VPWidenIntrinsicSC: + return cast(R); + case VPDef::VPWidenCallSC: + return cast(R); + case VPDef::VPWidenSelectSC: + return cast(R); + case VPDef::VPReplicateSC: + return cast(R); + case VPDef::VPInterleaveSC: + case VPDef::VPInterleaveEVLSC: + return cast(R); + case VPDef::VPWidenLoadSC: + case VPDef::VPWidenLoadEVLSC: + case VPDef::VPWidenStoreSC: + case VPDef::VPWidenStoreEVLSC: + return cast(R); + default: + llvm_unreachable("invalid recipe for VPIRMetadata cast"); + } +} +} // namespace detail + +/// Support casting from VPRecipeBase -> VPIRMetadata, by down-casting to the +/// recipe types implementing VPIRMetadata. Used by cast<>, dyn_cast<> & co. +template +struct CastInfoVPIRMetadata : public CastIsPossible { + static inline bool isPossible(SrcTy R) { + // NOTE: Each recipe inheriting from VPIRMetadata must be listed here and + // also handled in castToVPIRMetadata. + return isa(R); + } + + using RetTy = DstTy *; + + /// doCast is used by cast<>. + static inline RetTy doCast(SrcTy R) { + return detail::castToVPIRMetadata(R); + } + + /// doCastIfPossible is used by dyn_cast<>. + static inline RetTy doCastIfPossible(SrcTy R) { + if (!isPossible(R)) + return nullptr; + return doCast(R); + } +}; +template <> +struct CastInfo + : CastInfoVPIRMetadata {}; +template <> +struct CastInfo + : CastInfoVPIRMetadata {}; + /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It /// holds a sequence of zero or more VPRecipe's each representing a sequence of /// output IR instructions. All PHI-like recipes must come before any non-PHI recipes. diff --git a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp index 0e76c64f09f59..3842ba235ead3 100644 --- a/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp +++ b/llvm/unittests/Transforms/Vectorize/VPlanTest.cpp @@ -996,7 +996,7 @@ TEST_F(VPRecipeTest, CastVPInstructionToVPUser) { VPValue *Op2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2)); VPInstruction Recipe(Instruction::Add, {Op1, Op2}); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); } TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) { @@ -1011,7 +1011,7 @@ TEST_F(VPRecipeTest, CastVPWidenRecipeToVPUser) { Args.push_back(Op2); VPWidenRecipe WidenR(*AI, Args, VPIRMetadata(), DebugLoc()); - checkVPRecipeCastImpl(&WidenR); + checkVPRecipeCastImpl(&WidenR); delete AI; } @@ -1030,7 +1030,7 @@ TEST_F(VPRecipeTest, CastVPWidenCallRecipeToVPUserAndVPDef) { Args.push_back(CalledFn); VPWidenCallRecipe Recipe(Call, Fn, Args); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); VPValue *VPV = &Recipe; EXPECT_TRUE(VPV->getDefiningRecipe()); @@ -1056,7 +1056,8 @@ TEST_F(VPRecipeTest, CastVPWidenSelectRecipeToVPUserAndVPDef) { VPWidenSelectRecipe WidenSelectR(*SelectI, make_range(Args.begin(), Args.end())); - checkVPRecipeCastImpl(&WidenSelectR); + checkVPRecipeCastImpl( + &WidenSelectR); VPValue *VPV = &WidenSelectR; EXPECT_EQ(&WidenSelectR, VPV->getDefiningRecipe()); @@ -1094,7 +1095,7 @@ TEST_F(VPRecipeTest, CastVPWidenCastRecipeToVPUser) { VPValue *Op1 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 1)); VPWidenCastRecipe Recipe(Instruction::ZExt, Op1, Int64, *Cast, {}); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); delete Cast; } @@ -1105,7 +1106,7 @@ TEST_F(VPRecipeTest, CastVPWidenIntrinsicRecipeToVPUser) { VPValue *Op2 = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2)); VPWidenIntrinsicRecipe Recipe(Intrinsic::smax, {Op1, Op2}, Int32); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); } TEST_F(VPRecipeTest, CastVPBlendRecipeToVPUser) { @@ -1135,7 +1136,7 @@ TEST_F(VPRecipeTest, CastVPInterleaveRecipeToVPUser) { InterleaveGroup IG(4, false, Align(4)); VPInterleaveRecipe Recipe(&IG, Addr, {}, Mask, false, {}, DebugLoc()); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); } TEST_F(VPRecipeTest, CastVPReplicateRecipeToVPUser) { @@ -1151,7 +1152,7 @@ TEST_F(VPRecipeTest, CastVPReplicateRecipeToVPUser) { auto *Call = CallInst::Create(FTy, PoisonValue::get(FTy)); VPReplicateRecipe Recipe(Call, make_range(Args.begin(), Args.end()), true); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); delete Call; } @@ -1175,7 +1176,7 @@ TEST_F(VPRecipeTest, CastVPWidenMemoryRecipeToVPUserAndVPDef) { VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2)); VPWidenLoadRecipe Recipe(*Load, Addr, Mask, true, false, {}, {}); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); VPValue *VPV = Recipe.getVPSingleValue(); EXPECT_TRUE(isa(VPV->getDefiningRecipe())); @@ -1194,7 +1195,7 @@ TEST_F(VPRecipeTest, CastVPInterleaveEVLRecipeToVPUser) { VPInterleaveRecipe BaseRecipe(&IG, Addr, {}, Mask, false, {}, DebugLoc()); VPInterleaveEVLRecipe Recipe(BaseRecipe, *EVL, Mask); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); } TEST_F(VPRecipeTest, CastVPWidenLoadEVLRecipeToVPUser) { @@ -1209,7 +1210,7 @@ TEST_F(VPRecipeTest, CastVPWidenLoadEVLRecipeToVPUser) { VPWidenLoadRecipe BaseLoad(*Load, Addr, Mask, true, false, {}, {}); VPWidenLoadEVLRecipe Recipe(BaseLoad, Addr, *EVL, Mask); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); delete Load; } @@ -1225,7 +1226,7 @@ TEST_F(VPRecipeTest, CastVPWidenStoreRecipeToVPUser) { VPValue *Mask = Plan.getOrAddLiveIn(ConstantInt::get(Int32, 2)); VPWidenStoreRecipe Recipe(*Store, Addr, StoredVal, Mask, true, false, {}, {}); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); delete Store; } @@ -1244,7 +1245,7 @@ TEST_F(VPRecipeTest, CastVPWidenStoreEVLRecipeToVPUser) { {}); VPWidenStoreEVLRecipe Recipe(BaseStore, Addr, *EVL, Mask); - checkVPRecipeCastImpl(&Recipe); + checkVPRecipeCastImpl(&Recipe); delete Store; }