From 3aa97359238aabd0394c4c4914c4f27095f357b1 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Thu, 18 Sep 2025 12:35:02 +0100 Subject: [PATCH 1/3] [VPlanPatternMatch] Introduce m_APInt --- .../Transforms/Vectorize/VPlanPatternMatch.h | 26 +++++++++++++++++++ .../Transforms/Vectorize/VPlanTransforms.cpp | 9 ++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 8f9ce7a74b58b..7da75473ba00e 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -176,6 +176,32 @@ inline int_pred_ty m_ZeroInt() { /// For vectors, this includes constants with undefined elements. inline int_pred_ty m_One() { return int_pred_ty(); } +struct apint_match { + const APInt *&Res; + + apint_match(const APInt *&Res) : Res(Res) {} + + bool match(VPValue *VPV) const { + if (!VPV->isLiveIn()) + return false; + Value *V = VPV->getLiveInIRValue(); + if (!V) + return false; + const auto *CI = dyn_cast(V); + if (!CI && V->getType()->isVectorTy()) + if (const auto *C = dyn_cast(V)) + CI = dyn_cast_or_null( + C->getSplatValue(/*AllowPoison=*/false)); + if (!CI) + return false; + Res = &CI->getValue(); + return true; + } +}; + +/// Match an APInt, capturing it if we match. +inline apint_match m_APInt(const APInt *&C) { return C; } + /// Matching combinators template struct match_combine_or { LTy L; diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 9996b0167edcb..c71655f08bf49 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1592,11 +1592,12 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF, m_ActiveLaneMask(m_VPValue(Index), m_VPValue(), m_VPValue())); assert(Index && "Expected index from ActiveLaneMask instruction"); - auto *II = dyn_cast(Index); - if (II && II->getOpcode() == VPInstruction::CanonicalIVIncrementForPart) { - auto Part = cast(II->getOperand(1)->getLiveInIRValue()); + const APInt *Part; + if (match(Index, + m_VPInstruction( + m_VPValue(), m_APInt(Part)))) Phis[Part->getZExtValue()] = Phi; - } else + else // Anything other than a CanonicalIVIncrementForPart is part 0 Phis[0] = Phi; } From 784ca147f8b459351c709e50c8beaa827e1df277 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Thu, 18 Sep 2025 13:00:15 +0100 Subject: [PATCH 2/3] [VPlan/PM] Introduce m_ConstantInt --- .../Transforms/Vectorize/VPlanPatternMatch.h | 22 +++++++++++++++++-- .../Transforms/Vectorize/VPlanTransforms.cpp | 6 ++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index 7da75473ba00e..ae7b8d696a7ee 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -199,8 +199,26 @@ struct apint_match { } }; -/// Match an APInt, capturing it if we match. -inline apint_match m_APInt(const APInt *&C) { return C; } +struct bind_const_intval_ty { + uint64_t &VR; + + bind_const_intval_ty(uint64_t &V) : VR(V) {} + + template bool match(ITy *V) const { + const APInt *ConstInt; + if (!apint_match(ConstInt).match(V)) + return false; + if (auto C = ConstInt->tryZExtValue()) { + VR = *C; + return true; + } + return false; + } +}; + +/// Match a plain integer constant no wider than 64-bits, capturing it if we +/// match. +inline bind_const_intval_ty m_ConstantInt(uint64_t &C) { return C; } /// Matching combinators template struct match_combine_or { diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index c71655f08bf49..ea3df95c8e5be 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1592,11 +1592,11 @@ static bool tryToReplaceALMWithWideALM(VPlan &Plan, ElementCount VF, m_ActiveLaneMask(m_VPValue(Index), m_VPValue(), m_VPValue())); assert(Index && "Expected index from ActiveLaneMask instruction"); - const APInt *Part; + uint64_t Part; if (match(Index, m_VPInstruction( - m_VPValue(), m_APInt(Part)))) - Phis[Part->getZExtValue()] = Phi; + m_VPValue(), m_ConstantInt(Part)))) + Phis[Part] = Phi; else // Anything other than a CanonicalIVIncrementForPart is part 0 Phis[0] = Phi; From b078dc2a12e3e7c1ecfd97fc3cbbf0c9754abf3b Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Thu, 18 Sep 2025 16:18:04 +0100 Subject: [PATCH 3/3] [VPlan/PM] Remove indirection, strip dead code --- .../Transforms/Vectorize/VPlanPatternMatch.h | 31 +++++-------------- 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h index ae7b8d696a7ee..35cba04b66cc7 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h +++ b/llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h @@ -176,10 +176,10 @@ inline int_pred_ty m_ZeroInt() { /// For vectors, this includes constants with undefined elements. inline int_pred_ty m_One() { return int_pred_ty(); } -struct apint_match { - const APInt *&Res; +struct bind_const_int { + uint64_t &Res; - apint_match(const APInt *&Res) : Res(Res) {} + bind_const_int(uint64_t &Res) : Res(Res) {} bool match(VPValue *VPV) const { if (!VPV->isLiveIn()) @@ -187,29 +187,12 @@ struct apint_match { Value *V = VPV->getLiveInIRValue(); if (!V) return false; + assert(!V->getType()->isVectorTy() && "Unexpected vector live-in"); const auto *CI = dyn_cast(V); - if (!CI && V->getType()->isVectorTy()) - if (const auto *C = dyn_cast(V)) - CI = dyn_cast_or_null( - C->getSplatValue(/*AllowPoison=*/false)); if (!CI) return false; - Res = &CI->getValue(); - return true; - } -}; - -struct bind_const_intval_ty { - uint64_t &VR; - - bind_const_intval_ty(uint64_t &V) : VR(V) {} - - template bool match(ITy *V) const { - const APInt *ConstInt; - if (!apint_match(ConstInt).match(V)) - return false; - if (auto C = ConstInt->tryZExtValue()) { - VR = *C; + if (auto C = CI->getValue().tryZExtValue()) { + Res = *C; return true; } return false; @@ -218,7 +201,7 @@ struct bind_const_intval_ty { /// Match a plain integer constant no wider than 64-bits, capturing it if we /// match. -inline bind_const_intval_ty m_ConstantInt(uint64_t &C) { return C; } +inline bind_const_int m_ConstantInt(uint64_t &C) { return C; } /// Matching combinators template struct match_combine_or {