Skip to content

Commit

Permalink
[LoopVectorize] Fix strict reductions where VF = 1
Browse files Browse the repository at this point in the history 
Currently we will allow loops with a fixed width VF of 1 to vectorize
if the -enable-strict-reductions flag is set. However, the loop vectorizer
will not use ordered reductions if `VF.isScalar()` and the resulting
vectorized loop will be out of order.

This patch removes `VF.isVector()` when checking if ordered reductions
should be used. Also, instead of converting the FAdds to reductions if the
VF = 1, operands of the FAdds are changed such that the order is preserved.

Reviewed By: david-arm

Differential Revision: https://reviews.llvm.org/D104533
  • Loading branch information
@kmclaughlin-arm
kmclaughlin-arm committed Jun 28, 2021
1 parent 80aa7e1 commit f996725
Show file tree
Hide file tree
Showing 3 changed files with 103 additions and 22 deletions.
3 changes: 2 additions & 1 deletion llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
View file
Expand Up @@ -356,7 +356,8 @@ class LoopVectorizationPlanner {
/// reductions, with one operand being vector and the other being the scalar
/// reduction chain.
void adjustRecipesForInLoopReductions(VPlanPtr &Plan,
VPRecipeBuilder &RecipeBuilder);
VPRecipeBuilder &RecipeBuilder,
ElementCount MinVF);
};

} // namespace llvm
Expand Down
25 changes: 15 additions & 10 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Expand Up @@ -4344,8 +4344,7 @@ void InnerLoopVectorizer::fixReduction(VPWidenPHIRecipe *PhiR,
// any loop invariant values.
BasicBlock *VectorLoopLatch = LI->getLoopFor(LoopVectorBody)->getLoopLatch();

bool IsOrdered = State.VF.isVector() && IsInLoopReductionPhi &&
Cost->useOrderedReductions(RdxDesc);
bool IsOrdered = IsInLoopReductionPhi && Cost->useOrderedReductions(RdxDesc);

for (unsigned Part = 0; Part < UF; ++Part) {
if (IsOrdered && Part > 0)
Expand Down Expand Up @@ -4759,8 +4758,7 @@ void InnerLoopVectorizer::widenPHIInstruction(Instruction *PN,
Type *VecTy =
ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);

bool IsOrdered = State.VF.isVector() &&
Cost->isInLoopReduction(cast<PHINode>(PN)) &&
bool IsOrdered = Cost->isInLoopReduction(cast<PHINode>(PN)) &&
Cost->useOrderedReductions(*RdxDesc);
unsigned LastPartForNewPhi = IsOrdered ? 1 : State.UF;
for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
Expand Down Expand Up @@ -9280,8 +9278,7 @@ VPlanPtr LoopVectorizationPlanner::buildVPlanWithVPRecipes(
}

// Adjust the recipes for any inloop reductions.
if (Range.Start.isVector())
adjustRecipesForInLoopReductions(Plan, RecipeBuilder);
adjustRecipesForInLoopReductions(Plan, RecipeBuilder, Range.Start);

// Finally, if tail is folded by masking, introduce selects between the phi
// and the live-out instruction of each reduction, at the end of the latch.
Expand Down Expand Up @@ -9356,12 +9353,15 @@ VPlanPtr LoopVectorizationPlanner::buildVPlan(VFRange &Range) {
// reductions, with one operand being vector and the other being the scalar
// reduction chain.
void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder) {
VPlanPtr &Plan, VPRecipeBuilder &RecipeBuilder, ElementCount MinVF) {
for (auto &Reduction : CM.getInLoopReductionChains()) {
PHINode *Phi = Reduction.first;
RecurrenceDescriptor &RdxDesc = Legal->getReductionVars()[Phi];
const SmallVector<Instruction *, 4> &ReductionOperations = Reduction.second;

if (MinVF.isScalar() && !CM.useOrderedReductions(RdxDesc))
continue;

// ReductionOperations are orders top-down from the phi's use to the
// LoopExitValue. We keep a track of the previous item (the Chain) to tell
// which of the two operands will remain scalar and which will be reduced.
Expand All @@ -9378,7 +9378,7 @@ void LoopVectorizationPlanner::adjustRecipesForInLoopReductions(
"Expected to replace a VPWidenSelectSC");
FirstOpId = 1;
} else {
assert(isa<VPWidenRecipe>(WidenRecipe) &&
assert((MinVF.isScalar() || isa<VPWidenRecipe>(WidenRecipe)) &&
"Expected to replace a VPWidenSC");
FirstOpId = 0;
}
Expand Down Expand Up @@ -9527,8 +9527,13 @@ void VPReductionRecipe::execute(VPTransformState &State) {
Value *NewRed;
Value *NextInChain;
if (IsOrdered) {
NewRed = createOrderedReduction(State.Builder, *RdxDesc, NewVecOp,
PrevInChain);
if (State.VF.isVector())
NewRed = createOrderedReduction(State.Builder, *RdxDesc, NewVecOp,
PrevInChain);
else
NewRed = State.Builder.CreateBinOp(
(Instruction::BinaryOps)getUnderlyingInstr()->getOpcode(),
PrevInChain, NewVecOp);
PrevInChain = NewRed;
} else {
PrevInChain = State.get(getChainOp(), Part);
Expand Down
97 changes: 86 additions & 11 deletions llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
View file
Expand Up @@ -693,14 +693,89 @@ for.end:
ret float %add6
}

!0 = distinct !{!0, !4, !7, !9}
!1 = distinct !{!1, !4, !8, !9}
!2 = distinct !{!2, !5, !7, !9}
!3 = distinct !{!3, !6, !7, !9, !10}
!4 = !{!"llvm.loop.vectorize.width", i32 8}
!5 = !{!"llvm.loop.vectorize.width", i32 4}
!6 = !{!"llvm.loop.vectorize.width", i32 2}
!7 = !{!"llvm.loop.interleave.count", i32 1}
!8 = !{!"llvm.loop.interleave.count", i32 4}
!9 = !{!"llvm.loop.vectorize.enable", i1 true}
!10 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}
; Test reductions for a VF of 1 and a UF > 1.
define float @fadd_scalar_vf(float* noalias nocapture readonly %a, i64 %n) {
; CHECK-ORDERED-LABEL: @fadd_scalar_vf
; CHECK-ORDERED: vector.body
; CHECK-ORDERED: %[[VEC_PHI:.*]] = phi float [ 0.000000e+00, {{.*}} ], [ %[[FADD4:.*]], %vector.body ]
; CHECK-ORDERED: %[[LOAD1:.*]] = load float, float*
; CHECK-ORDERED: %[[LOAD2:.*]] = load float, float*
; CHECK-ORDERED: %[[LOAD3:.*]] = load float, float*
; CHECK-ORDERED: %[[LOAD4:.*]] = load float, float*
; CHECK-ORDERED: %[[FADD1:.*]] = fadd float %[[VEC_PHI]], %[[LOAD1]]
; CHECK-ORDERED: %[[FADD2:.*]] = fadd float %[[FADD1]], %[[LOAD2]]
; CHECK-ORDERED: %[[FADD3:.*]] = fadd float %[[FADD2]], %[[LOAD3]]
; CHECK-ORDERED: %[[FADD4]] = fadd float %[[FADD3]], %[[LOAD4]]
; CHECK-ORDERED-NOT: call float @llvm.vector.reduce.fadd
; CHECK-ORDERED: scalar.ph
; CHECK-ORDERED: %[[MERGE_RDX:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[FADD4]], %middle.block ]
; CHECK-ORDERED: for.body
; CHECK-ORDERED: %[[SUM_PHI:.*]] = phi float [ %[[MERGE_RDX]], %scalar.ph ], [ %[[FADD5:.*]], %for.body ]
; CHECK-ORDERED: %[[LOAD5:.*]] = load float, float*
; CHECK-ORDERED: %[[FADD5]] = fadd float %[[LOAD5]], %[[SUM_PHI]]
; CHECK-ORDERED: for.end
; CHECK-ORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD5]], %for.body ], [ %[[FADD4]], %middle.block ]
; CHECK-ORDERED: ret float %[[RES_PHI]]

; CHECK-UNORDERED-LABEL: @fadd_scalar_vf
; CHECK-UNORDERED: vector.body
; CHECK-UNORDERED: %[[VEC_PHI1:.*]] = phi float [ 0.000000e+00, %vector.ph ], [ %[[FADD1:.*]], %vector.body ]
; CHECK-UNORDERED: %[[VEC_PHI2:.*]] = phi float [ -0.000000e+00, %vector.ph ], [ %[[FADD2:.*]], %vector.body ]
; CHECK-UNORDERED: %[[VEC_PHI3:.*]] = phi float [ -0.000000e+00, %vector.ph ], [ %[[FADD3:.*]], %vector.body ]
; CHECK-UNORDERED: %[[VEC_PHI4:.*]] = phi float [ -0.000000e+00, %vector.ph ], [ %[[FADD4:.*]], %vector.body ]
; CHECK-UNORDERED: %[[LOAD1:.*]] = load float, float*
; CHECK-UNORDERED: %[[LOAD2:.*]] = load float, float*
; CHECK-UNORDERED: %[[LOAD3:.*]] = load float, float*
; CHECK-UNORDERED: %[[LOAD4:.*]] = load float, float*
; CHECK-UNORDERED: %[[FADD1]] = fadd float %[[LOAD1]], %[[VEC_PHI1]]
; CHECK-UNORDERED: %[[FADD2]] = fadd float %[[LOAD2]], %[[VEC_PHI2]]
; CHECK-UNORDERED: %[[FADD3]] = fadd float %[[LOAD3]], %[[VEC_PHI3]]
; CHECK-UNORDERED: %[[FADD4]] = fadd float %[[LOAD4]], %[[VEC_PHI4]]
; CHECK-UNORDERED-NOT: call float @llvm.vector.reduce.fadd
; CHECK-UNORDERED: middle.block
; CHECK-UNORDERED: %[[BIN_RDX1:.*]] = fadd float %[[FADD2]], %[[FADD1]]
; CHECK-UNORDERED: %[[BIN_RDX2:.*]] = fadd float %[[FADD3]], %[[BIN_RDX1]]
; CHECK-UNORDERED: %[[BIN_RDX3:.*]] = fadd float %[[FADD4]], %[[BIN_RDX2]]
; CHECK-UNORDERED: scalar.ph
; CHECK-UNORDERED: %[[MERGE_RDX:.*]] = phi float [ 0.000000e+00, %entry ], [ %[[BIN_RDX3]], %middle.block ]
; CHECK-UNORDERED: for.body
; CHECK-UNORDERED: %[[SUM_PHI:.*]] = phi float [ %[[MERGE_RDX]], %scalar.ph ], [ %[[FADD5:.*]], %for.body ]
; CHECK-UNORDERED: %[[LOAD5:.*]] = load float, float*
; CHECK-UNORDERED: %[[FADD5]] = fadd float %[[LOAD5]], %[[SUM_PHI]]
; CHECK-UNORDERED: for.end
; CHECK-UNORDERED: %[[RES_PHI:.*]] = phi float [ %[[FADD5]], %for.body ], [ %[[BIN_RDX3]], %middle.block ]
; CHECK-UNORDERED: ret float %[[RES_PHI]]

; CHECK-NOT-VECTORIZED-LABEL: @fadd_scalar_vf
; CHECK-NOT-VECTORIZED-NOT: @vector.body

entry:
br label %for.body

for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%sum.07 = phi float [ 0.000000e+00, %entry ], [ %add, %for.body ]
%arrayidx = getelementptr inbounds float, float* %a, i64 %iv
%0 = load float, float* %arrayidx, align 4
%add = fadd float %0, %sum.07
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, %n
br i1 %exitcond.not, label %for.end, label %for.body, !llvm.loop !4

for.end:
ret float %add
}

!0 = distinct !{!0, !5, !9, !11}
!1 = distinct !{!1, !5, !10, !11}
!2 = distinct !{!2, !6, !9, !11}
!3 = distinct !{!3, !7, !9, !11, !12}
!4 = distinct !{!4, !8, !10, !11}
!5 = !{!"llvm.loop.vectorize.width", i32 8}
!6 = !{!"llvm.loop.vectorize.width", i32 4}
!7 = !{!"llvm.loop.vectorize.width", i32 2}
!8 = !{!"llvm.loop.vectorize.width", i32 1}
!9 = !{!"llvm.loop.interleave.count", i32 1}
!10 = !{!"llvm.loop.interleave.count", i32 4}
!11 = !{!"llvm.loop.vectorize.enable", i1 true}
!12 = !{!"llvm.loop.vectorize.predicate.enable", i1 true}

0 comments on commit f996725

Please sign in to comment.