Skip to content

Commit

Permalink
[LoopVectorize] Check the number of uses of an FAdd before classifyin…
Browse files Browse the repository at this point in the history
…g as ordered

checkOrderedReductions looks for Phi nodes which can be classified as in-order,
meaning they can be vectorised without unsafe math. In order to vectorise the
reduction it should also be classified as in-loop by getReductionOpChain, which
checks that the reduction has two uses.

In this patch, a similar check is added to checkOrderedReductions so that we
now return false if there are more than two uses of the FAdd instruction.
This fixes PR52515.

Reviewed By: fhahn, david-arm

Differential Revision: https://reviews.llvm.org/D114002
  • Loading branch information
kmclaughlin-arm committed Nov 18, 2021
1 parent e6c66ef commit ff64b29
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 1 deletion.
5 changes: 4 additions & 1 deletion llvm/lib/Analysis/IVDescriptors.cpp
Expand Up @@ -198,7 +198,10 @@ static bool checkOrderedReduction(RecurKind Kind, Instruction *ExactFPMathInst,
if (Kind != RecurKind::FAdd)
return false;

if (Exit->getOpcode() != Instruction::FAdd || Exit != ExactFPMathInst)
// Ensure the exit instruction is an FAdd, and that it only has one user
// other than the reduction PHI
if (Exit->getOpcode() != Instruction::FAdd || Exit->hasNUsesOrMore(3) ||
Exit != ExactFPMathInst)
return false;

// The only pattern accepted is the one in which the reduction PHI
Expand Down
36 changes: 36 additions & 0 deletions llvm/test/Transforms/LoopVectorize/AArch64/strict-fadd.ll
Expand Up @@ -931,6 +931,42 @@ exit:
ret double %res
}

; We should not mark the fadd as an ordered reduction here as there are
; more than 2 uses of the instruction
define float @fadd_multiple_use(i64 %n) {
; CHECK-ORDERED-LABEL: @fadd_multiple_use
; CHECK-ORDERED-LABEL-NOT: vector.body

; CHECK-UNORDERED-LABEL: @fadd_multiple_use
; CHECK-UNORDERED-LABEL-NOT: vector.body

; CHECK-NOT-VECTORIZED-LABEL: @fadd_multiple_use
; CHECK-NOT-VECTORIZED-NOT: vector.body
entry:
br label %for.body

for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next2, %bb2 ]
%red = phi float [ 0.0, %entry ], [ %fadd, %bb2 ]
%phi1 = phi i64 [ 0, %entry ], [ %iv.next, %bb2 ]
%fadd = fadd float %red, 1.000000e+00
%iv.next = add nsw i64 %phi1, 1
%cmp = icmp ult i64 %iv, %n
br i1 %cmp, label %bb2, label %bb1

bb1:
%phi2 = phi float [ %fadd, %for.body ]
ret float %phi2

bb2:
%iv.next2 = add nuw nsw i64 %iv, 1
br i1 false, label %for.end, label %for.body

for.end:
%phi3 = phi float [ %fadd, %bb2 ]
ret float %phi3
}

!0 = distinct !{!0, !5, !9, !11}
!1 = distinct !{!1, !5, !10, !11}
!2 = distinct !{!2, !6, !9, !11}
Expand Down

0 comments on commit ff64b29

Please sign in to comment.