Skip to content

[LV] Create in-loop sub reductions #147026

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 22 commits into from
Aug 12, 2025
Merged

[LV] Create in-loop sub reductions #147026

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
6171fd6
[LV] Create in-loop sub reductions
SamTebbs33 Jul 1, 2025
93dd762
Use empty flags for negation
SamTebbs33 Jul 16, 2025
52a8138
Improve test comment
SamTebbs33 Jul 16, 2025
2f7a77f
Add RecurKind::Sub
SamTebbs33 Jul 16, 2025
b38b1cb
Address review
SamTebbs33 Jul 18, 2025
75baa8e
Fix inloop reductions
SamTebbs33 Jul 21, 2025
0669c78
Remove CurrentLinkI opcode check
SamTebbs33 Jul 21, 2025
e855872
Fix interleaving case
SamTebbs33 Jul 22, 2025
35b6b9d
format
SamTebbs33 Jul 22, 2025
c659696
Improve codegen
SamTebbs33 Jul 29, 2025
2b8e239
Address review
SamTebbs33 Jul 30, 2025
bdf2604
format
SamTebbs33 Jul 30, 2025
83f029c
Improve mixed recurrence kind cases
SamTebbs33 Jul 31, 2025
6978e95
Remove check in isCorrectOpcode
SamTebbs33 Jul 31, 2025
f53accf
Update test comment and set start value to 3
SamTebbs33 Aug 1, 2025
0033606
Add Sub cases to SLPVectorizer
SamTebbs33 Aug 1, 2025
3ae3e67
Add test for mixed inloop reductions
SamTebbs33 Aug 4, 2025
0f5cb6d
Format and remove check from isCorrectOpcode
SamTebbs33 Aug 5, 2025
bd5c474
Make RecurKind::Sub unreachable in SLPVectorizer.cpp
SamTebbs33 Aug 6, 2025
6a6bb86
Add RecurKind::AddChainWithSubs
SamTebbs33 Aug 7, 2025
c32b0e1
Simplify isRecurrenceInstr, getReductionOpChain and remove a case in SLP
SamTebbs33 Aug 8, 2025
340003b
Revert partial-reduce-chained.ll changes
SamTebbs33 Aug 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions llvm/include/llvm/Analysis/IVDescriptors.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ enum class RecurKind {
// clang-format off
None, ///< Not a recurrence.
Add, ///< Sum of integers.
Sub, ///< Subtraction of integers
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You will need to update a number of places with switches over recurrence kind, in SLPVectorizer and possibly others

Probably also should have a SLP test to make sure its handled correctly.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I've added RecurKind::Sub cases to the SLP vectorizer and added a test. I don't think adding proper sub reduction support to SLP is in scope of this PR so haven't added full support.

AddChainWithSubs, ///< A chain of adds and subs
Mul, ///< Product of integers.
Or, ///< Bitwise or logical OR of integers.
And, ///< Bitwise or logical AND of integers.
Expand Down
28 changes: 26 additions & 2 deletions llvm/lib/Analysis/IVDescriptors.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ bool RecurrenceDescriptor::isIntegerRecurrenceKind(RecurKind Kind) {
switch (Kind) {
default:
break;
case RecurKind::AddChainWithSubs:
case RecurKind::Sub:
case RecurKind::Add:
case RecurKind::Mul:
case RecurKind::Or:
Expand Down Expand Up @@ -897,8 +899,11 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
case Instruction::PHI:
return InstDesc(I, Prev.getRecKind(), Prev.getExactFPMathInst());
case Instruction::Sub:
return InstDesc(
Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs, I);
case Instruction::Add:
return InstDesc(Kind == RecurKind::Add, I);
return InstDesc(
Kind == RecurKind::Add || Kind == RecurKind::AddChainWithSubs, I);
case Instruction::Mul:
return InstDesc(Kind == RecurKind::Mul, I);
case Instruction::And:
Expand All @@ -917,7 +922,8 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(
I->hasAllowReassoc() ? nullptr : I);
case Instruction::Select:
if (Kind == RecurKind::FAdd || Kind == RecurKind::FMul ||
Kind == RecurKind::Add || Kind == RecurKind::Mul)
Kind == RecurKind::Add || Kind == RecurKind::Mul ||
Kind == RecurKind::Sub || Kind == RecurKind::AddChainWithSubs)
return isConditionalRdxPattern(I);
if (isFindIVRecurrenceKind(Kind) && SE)
return isFindIVPattern(Kind, L, OrigPhi, I, *SE);
Expand Down Expand Up @@ -1003,6 +1009,17 @@ bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop,
LLVM_DEBUG(dbgs() << "Found an ADD reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::Sub, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a SUB reduction PHI." << *Phi << "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::AddChainWithSubs, TheLoop, FMF, RedDes,
DB, AC, DT, SE)) {
LLVM_DEBUG(dbgs() << "Found a chained ADD-SUB reduction PHI." << *Phi
<< "\n");
return true;
}
if (AddReductionVar(Phi, RecurKind::Mul, TheLoop, FMF, RedDes, DB, AC, DT,
SE)) {
LLVM_DEBUG(dbgs() << "Found a MUL reduction PHI." << *Phi << "\n");
Expand Down Expand Up @@ -1201,6 +1218,9 @@ bool RecurrenceDescriptor::isFixedOrderRecurrence(PHINode *Phi, Loop *TheLoop,

unsigned RecurrenceDescriptor::getOpcode(RecurKind Kind) {
switch (Kind) {
case RecurKind::Sub:
return Instruction::Sub;
case RecurKind::AddChainWithSubs:
case RecurKind::Add:
return Instruction::Add;
case RecurKind::Mul:
Expand Down Expand Up @@ -1288,6 +1308,10 @@ RecurrenceDescriptor::getReductionOpChain(PHINode *Phi, Loop *L) const {
if (isFMulAddIntrinsic(Cur))
return true;

if (Cur->getOpcode() == Instruction::Sub &&
Kind == RecurKind::AddChainWithSubs)
return true;

return Cur->getOpcode() == getOpcode();
};

Expand Down
2 changes: 2 additions & 0 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5151,6 +5151,8 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction(
return false;

switch (RdxDesc.getRecurrenceKind()) {
case RecurKind::Sub:
case RecurKind::AddChainWithSubs:
Comment on lines +5154 to +5155
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just a heads up, this caused some regressions on RISC-V because sub reductions are now a different type. We have a fix up to update the TTI hook here #154753 but it might be worth looking at the other targets/X86 to see if they also need to have their isLegalToVectorizeReduction updated?

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

From what I can see only AArch64 and RISC-V targets have this target hook implemented.

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for fixing that. Sander is right about it only being implemented for AArch64 and RISC-V. It just returns a blanket true by default.

case RecurKind::Add:
case RecurKind::FAdd:
case RecurKind::And:
Expand Down
4 changes: 4 additions & 0 deletions llvm/lib/Transforms/Utils/LoopUtils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,8 @@ constexpr Intrinsic::ID llvm::getReductionIntrinsicID(RecurKind RK) {
switch (RK) {
default:
llvm_unreachable("Unexpected recurrence kind");
case RecurKind::AddChainWithSubs:
case RecurKind::Sub:
case RecurKind::Add:
return Intrinsic::vector_reduce_add;
case RecurKind::Mul:
Expand Down Expand Up @@ -1301,6 +1303,8 @@ Value *llvm::createSimpleReduction(IRBuilderBase &Builder, Value *Src,
Builder.getFastMathFlags());
};
switch (RdxKind) {
case RecurKind::AddChainWithSubs:
case RecurKind::Sub:
case RecurKind::Add:
case RecurKind::Mul:
case RecurKind::And:
Expand Down
10 changes: 10 additions & 0 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9118,6 +9118,16 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
CurrentLinkI->getFastMathFlags());
LinkVPBB->insert(FMulRecipe, CurrentLink->getIterator());
VecOp = FMulRecipe;
} else if (PhiR->isInLoop() && Kind == RecurKind::AddChainWithSubs &&
CurrentLinkI->getOpcode() == Instruction::Sub) {
Type *PhiTy = PhiR->getUnderlyingValue()->getType();
auto *Zero = Plan->getOrAddLiveIn(ConstantInt::get(PhiTy, 0));
VPWidenRecipe *Sub = new VPWidenRecipe(
Instruction::Sub, {Zero, CurrentLink->getOperand(1)}, {},
VPIRMetadata(), CurrentLinkI->getDebugLoc());
Sub->setUnderlyingValue(CurrentLinkI);
LinkVPBB->insert(Sub, CurrentLink->getIterator());
VecOp = Sub;
} else {
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(Kind)) {
if (isa<VPWidenRecipe>(CurrentLink)) {
Expand Down
6 changes: 6 additions & 0 deletions llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23697,6 +23697,8 @@ class HorizontalReduction {
case RecurKind::FMinimum:
// res = vv
break;
case RecurKind::Sub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
Expand Down Expand Up @@ -23836,6 +23838,8 @@ class HorizontalReduction {
case RecurKind::FMinimum:
// res = vv
return VectorizedValue;
case RecurKind::Sub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
Expand Down Expand Up @@ -23940,6 +23944,8 @@ class HorizontalReduction {
auto *Scale = ConstantVector::get(Vals);
return Builder.CreateFMul(VectorizedValue, Scale);
}
case RecurKind::Sub:
case RecurKind::AddChainWithSubs:
case RecurKind::Mul:
case RecurKind::FMul:
case RecurKind::FMulAdd:
Expand Down
16 changes: 12 additions & 4 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -812,10 +812,18 @@ Value *VPInstruction::generate(VPTransformState &State) {
Value *RdxPart = RdxParts[Part];
if (RecurrenceDescriptor::isMinMaxRecurrenceKind(RK))
ReducedPartRdx = createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
else
ReducedPartRdx = Builder.CreateBinOp(
(Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK),
RdxPart, ReducedPartRdx, "bin.rdx");
else {
Instruction::BinaryOps Opcode;
// For sub-recurrences, each UF's reduction variable is already
// negative, we need to do: reduce.add(-acc_uf0 + -acc_uf1)
if (RK == RecurKind::Sub)
Opcode = Instruction::Add;
else
Opcode =
(Instruction::BinaryOps)RecurrenceDescriptor::getOpcode(RK);
ReducedPartRdx =
Builder.CreateBinOp(Opcode, RdxPart, ReducedPartRdx, "bin.rdx");
}
}
}

Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1742,7 +1742,8 @@ void VPlanTransforms::clearReductionWrapFlags(VPlan &Plan) {
if (!PhiR)
continue;
RecurKind RK = PhiR->getRecurrenceKind();
if (RK != RecurKind::Add && RK != RecurKind::Mul)
if (RK != RecurKind::Add && RK != RecurKind::Mul && RK != RecurKind::Sub &&
RK != RecurKind::AddChainWithSubs)
continue;

for (VPUser *U : collectUsersRecursively(PhiR))
Expand Down
Loading
Loading