[NARY-REASSOCIATE] Support reassociation of min/max

Support reassociation for min/max. With that we should be able to transform min(min(a, b), c) -> min(min(a, c), b) if min(a, c) is already available.

Reviewed By: mkazantsev

Differential Revision: https://reviews.llvm.org/D88287

llvm/include/llvm/IR/PatternMatch.h

@@ -1707,6 +1707,7 @@ m_Br(const Cond_t &C, const TrueBlock_t &T, const FalseBlock_t &F) {
 template <typename CmpInst_t, typename LHS_t, typename RHS_t, typename Pred_t,
           bool Commutable = false>
 struct MaxMin_match {
+  using PredType = Pred_t;
   LHS_t L;
   RHS_t R;
 

llvm/include/llvm/Transforms/Scalar/NaryReassociate.h

@@ -158,6 +158,19 @@ class NaryReassociatePass : public PassInfoMixin<NaryReassociatePass> {
   Instruction *findClosestMatchingDominator(const SCEV *CandidateExpr,
                                             Instruction *Dominatee);
 
+  // Try to match \p I as signed/unsigned Min/Max and reassociate it. \p
+  // OrigSCEV is set if \I matches Min/Max regardless whether resassociation is
+  // done or not. If reassociation was successful newly generated instruction is
+  // returned, otherwise nullptr.
+  template <typename PredT>
+  Instruction *matchAndReassociateMinOrMax(Instruction *I,
+                                           const SCEV *&OrigSCEV);
+
+  // Reassociate Min/Max.
+  template <typename MaxMinT>
+  Value *tryReassociateMinOrMax(Instruction *I, MaxMinT MaxMinMatch, Value *LHS,
+                                Value *RHS);
+
   // GetElementPtrInst implicitly sign-extends an index if the index is shorter
   // than the pointer size. This function returns whether Index is shorter than
   // GEP's pointer size, i.e., whether Index needs to be sign-extended in order

llvm/lib/Transforms/Scalar/NaryReassociate.cpp

@@ -80,6 +80,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -106,6 +107,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include <cassert>
 #include <cstdint>
 
@@ -269,6 +271,24 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
   return Changed;
 }
 
+template <typename PredT>
+Instruction *
+NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
+                                                 const SCEV *&OrigSCEV) {
+  Value *LHS = nullptr;
+  Value *RHS = nullptr;
+
+  auto MinMaxMatcher =
+      MaxMin_match<ICmpInst, bind_ty<Value>, bind_ty<Value>, PredT>(
+          m_Value(LHS), m_Value(RHS));
+  if (match(I, MinMaxMatcher)) {
+    OrigSCEV = SE->getSCEV(I);
+    return dyn_cast_or_null<Instruction>(
+        tryReassociateMinOrMax(I, MinMaxMatcher, LHS, RHS));
+  }
+  return nullptr;
+}
+
 Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
                                                  const SCEV *&OrigSCEV) {
 
@@ -284,10 +304,17 @@ Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
     OrigSCEV = SE->getSCEV(I);
     return tryReassociateGEP(cast<GetElementPtrInst>(I));
   default:
-    return nullptr;
+    break;
   }
 
-  llvm_unreachable("should not be reached");
+  // Try to match signed/unsigned Min/Max.
+  Instruction *ResI = nullptr;
+  if ((ResI = matchAndReassociateMinOrMax<umin_pred_ty>(I, OrigSCEV)) ||
+      (ResI = matchAndReassociateMinOrMax<smin_pred_ty>(I, OrigSCEV)) ||
+      (ResI = matchAndReassociateMinOrMax<umax_pred_ty>(I, OrigSCEV)) ||
+      (ResI = matchAndReassociateMinOrMax<smax_pred_ty>(I, OrigSCEV)))
+    return ResI;
+
   return nullptr;
 }
 
@@ -540,3 +567,72 @@ NaryReassociatePass::findClosestMatchingDominator(const SCEV *CandidateExpr,
   }
   return nullptr;
 }
+
+template <typename MaxMinT> static SCEVTypes convertToSCEVype(MaxMinT &MM) {
+  if (std::is_same<smax_pred_ty, typename MaxMinT::PredType>::value)
+    return scSMaxExpr;
+  else if (std::is_same<umax_pred_ty, typename MaxMinT::PredType>::value)
+    return scUMaxExpr;
+  else if (std::is_same<smin_pred_ty, typename MaxMinT::PredType>::value)
+    return scSMinExpr;
+  else if (std::is_same<umin_pred_ty, typename MaxMinT::PredType>::value)
+    return scUMinExpr;
+
+  llvm_unreachable("Can't convert MinMax pattern to SCEV type");
+  return scUnknown;
+}
+
+template <typename MaxMinT>
+Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
+                                                   MaxMinT MaxMinMatch,
+                                                   Value *LHS, Value *RHS) {
+  Value *A = nullptr, *B = nullptr;
+  MaxMinT m_MaxMin(m_Value(A), m_Value(B));
+  for (uint i = 0; i < 2; ++i) {
+    if (match(LHS, m_MaxMin)) {
+      const SCEV *AExpr = SE->getSCEV(A), *BExpr = SE->getSCEV(B);
+      const SCEV *RHSExpr = SE->getSCEV(RHS);
+      for (uint j = 0; j < 2; ++j) {
+        if (j == 0) {
+          if (BExpr == RHSExpr)
+            continue;
+          // Transform 'I = (A op B) op RHS' to 'I = (A op RHS) op B' on the
+          // first iteration.
+          std::swap(BExpr, RHSExpr);
+        } else {
+          if (AExpr == RHSExpr)
+            continue;
+          // Transform 'I = (A op RHS) op B' 'I = (B op RHS) op A' on the second
+          // iteration.
+          std::swap(AExpr, RHSExpr);
+        }
+
+        SCEVExpander Expander(*SE, *DL, "nary-reassociate");
+        SmallVector<const SCEV *, 2> Ops1{ BExpr, AExpr };
+        const SCEVTypes SCEVType = convertToSCEVype(m_MaxMin);
+        const SCEV *R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
+
+        Instruction *R1MinMax = findClosestMatchingDominator(R1Expr, I);
+
+        if (!R1MinMax)
+          continue;
+
+        LLVM_DEBUG(dbgs() << "NARY: Found common sub-expr: " << *R1MinMax
+                          << "\n");
+
+        R1Expr = SE->getUnknown(R1MinMax);
+        SmallVector<const SCEV *, 2> Ops2{ RHSExpr, R1Expr };
+        const SCEV *R2Expr = SE->getMinMaxExpr(SCEVType, Ops2);
+
+        Value *NewMinMax = Expander.expandCodeFor(R2Expr, I->getType(), I);
+        NewMinMax->setName(Twine(I->getName()).concat(".nary"));
+
+        LLVM_DEBUG(dbgs() << "NARY: Deleting:  " << *I << "\n"
+                          << "NARY: Inserting: " << *NewMinMax << "\n");
+        return NewMinMax;
+      }
+    }
+    std::swap(LHS, RHS);
+  }
+  return nullptr;
+}

llvm/test/Transforms/NaryReassociate/nary-smax.ll

@@ -10,11 +10,9 @@ define i32 @smax_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test1(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i32 [[SMAX2]], [[A]]
-; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[C3]], i32 [[SMAX2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp sgt i32 %a, %b
@@ -33,11 +31,9 @@ define i32 @smax_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test2(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[A]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[C2]], i32 [[A]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i32 [[B]], [[SMAX2]]
-; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[C3]], i32 [[B]], i32 [[SMAX2]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp sgt i32 %a, %b
@@ -54,9 +50,9 @@ define i32 @smax_test2(i32 %a, i32 %b, i32 %c) {
 define i32 @smax_test3(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test3(
 ; CHECK-NEXT:    [[SMAX1:%.*]] = call i32 @llvm.smax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
-; CHECK-NEXT:    [[SMAX2:%.*]] = call i32 @llvm.smax.i32(i32 [[B]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[SMAX3:%.*]] = call i32 @llvm.smax.i32(i32 [[SMAX2]], i32 [[A]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %smax1 = call i32 @llvm.smax.i32(i32 %a, i32 %b)
@@ -72,11 +68,9 @@ define i32 @umax_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test4(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sge i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX_OR_EQ2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sgt i32 [[SMAX_OR_EQ2]], [[A]]
-; CHECK-NEXT:    [[SMAX3:%.*]] = select i1 [[C3]], i32 [[SMAX_OR_EQ2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX1]], [[SMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp sgt i32 %a, %b
@@ -95,11 +89,9 @@ define i32 @smax_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smax_test5(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sge i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMAX_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sgt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sge i32 [[SMAX2]], [[A]]
-; CHECK-NEXT:    [[SMAX_OR_EQ3:%.*]] = select i1 [[C3]], i32 [[SMAX2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX_OR_EQ1]], [[SMAX_OR_EQ3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp sgt i32 [[SMAX_OR_EQ1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMAX_OR_EQ3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMAX_OR_EQ1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMAX_OR_EQ1]], [[SMAX_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp sge i32 %a, %b

llvm/test/Transforms/NaryReassociate/nary-smin.ll

@@ -10,11 +10,9 @@ define i32 @smin_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test1(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[SMIN2]], [[A]]
-; CHECK-NEXT:    [[SMIN3:%.*]] = select i1 [[C3]], i32 [[SMIN2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SMIN1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMIN1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp slt i32 %a, %b
@@ -33,11 +31,9 @@ define i32 @smin_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test2(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[A]], [[C:%.*]]
-; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[C2]], i32 [[A]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[B]], [[SMIN2]]
-; CHECK-NEXT:    [[SMIN3:%.*]] = select i1 [[C3]], i32 [[B]], i32 [[SMIN2]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SMIN1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMIN1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp slt i32 %a, %b
@@ -54,9 +50,9 @@ define i32 @smin_test2(i32 %a, i32 %b, i32 %c) {
 define i32 @smin_test3(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test3(
 ; CHECK-NEXT:    [[SMIN1:%.*]] = call i32 @llvm.smin.i32(i32 [[A:%.*]], i32 [[B:%.*]])
-; CHECK-NEXT:    [[SMIN2:%.*]] = call i32 @llvm.smin.i32(i32 [[B]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[SMIN3:%.*]] = call i32 @llvm.smin.i32(i32 [[SMIN2]], i32 [[A]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SMIN1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMIN1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %smin1 = call i32 @llvm.smin.i32(i32 %a, i32 %b)
@@ -72,11 +68,9 @@ define i32 @umin_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umin_test4(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp slt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMIN1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp sle i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMIN_OR_EQ2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp slt i32 [[SMIN_OR_EQ2]], [[A]]
-; CHECK-NEXT:    [[SMIN3:%.*]] = select i1 [[C3]], i32 [[SMIN_OR_EQ2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SMIN1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMIN3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMIN1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN1]], [[SMIN3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp slt i32 %a, %b
@@ -95,11 +89,9 @@ define i32 @smin_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @smin_test5(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp sle i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[SMIN_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp slt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[SMIN2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp sle i32 [[SMIN2]], [[A]]
-; CHECK-NEXT:    [[SMIN_OR_EQ3:%.*]] = select i1 [[C3]], i32 [[SMIN2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN_OR_EQ1]], [[SMIN_OR_EQ3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp slt i32 [[SMIN_OR_EQ1]], [[C:%.*]]
+; CHECK-NEXT:    [[SMIN_OR_EQ3_NARY:%.*]] = select i1 [[TMP1]], i32 [[SMIN_OR_EQ1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[SMIN_OR_EQ1]], [[SMIN_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp sle i32 %a, %b

llvm/test/Transforms/NaryReassociate/nary-umax.ll

@@ -10,11 +10,9 @@ define i32 @umax_test1(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test1(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ugt i32 [[UMAX2]], [[A]]
-; CHECK-NEXT:    [[UMAX3:%.*]] = select i1 [[C3]], i32 [[UMAX2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[UMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[UMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp ugt i32 %a, %b
@@ -33,11 +31,9 @@ define i32 @umax_test2(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test2(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[A]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX2:%.*]] = select i1 [[C2]], i32 [[A]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ugt i32 [[B]], [[UMAX2]]
-; CHECK-NEXT:    [[UMAX3:%.*]] = select i1 [[C3]], i32 [[B]], i32 [[UMAX2]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[UMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[UMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp ugt i32 %a, %b
@@ -54,9 +50,9 @@ define i32 @umax_test2(i32 %a, i32 %b, i32 %c) {
 define i32 @umax_test3(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test3(
 ; CHECK-NEXT:    [[UMAX1:%.*]] = call i32 @llvm.umax.i32(i32 [[A:%.*]], i32 [[B:%.*]])
-; CHECK-NEXT:    [[UMAX2:%.*]] = call i32 @llvm.umax.i32(i32 [[B]], i32 [[C:%.*]])
-; CHECK-NEXT:    [[UMAX3:%.*]] = call i32 @llvm.umax.i32(i32 [[UMAX2]], i32 [[A]])
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[UMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[UMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %umax1 = call i32 @llvm.umax.i32(i32 %a, i32 %b)
@@ -72,11 +68,9 @@ define i32 @umax_test4(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test4(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp ugt i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[UMAX1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp uge i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX_OR_EQ2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp ugt i32 [[UMAX_OR_EQ2]], [[A]]
-; CHECK-NEXT:    [[UMAX3:%.*]] = select i1 [[C3]], i32 [[UMAX_OR_EQ2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[UMAX1]], [[C:%.*]]
+; CHECK-NEXT:    [[UMAX3_NARY:%.*]] = select i1 [[TMP1]], i32 [[UMAX1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX1]], [[UMAX3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp ugt i32 %a, %b
@@ -95,11 +89,9 @@ define i32 @umax_test5(i32 %a, i32 %b, i32 %c) {
 ; CHECK-LABEL: @umax_test5(
 ; CHECK-NEXT:    [[C1:%.*]] = icmp uge i32 [[A:%.*]], [[B:%.*]]
 ; CHECK-NEXT:    [[UMAX_OR_EQ1:%.*]] = select i1 [[C1]], i32 [[A]], i32 [[B]]
-; CHECK-NEXT:    [[C2:%.*]] = icmp ugt i32 [[B]], [[C:%.*]]
-; CHECK-NEXT:    [[UMAX2:%.*]] = select i1 [[C2]], i32 [[B]], i32 [[C]]
-; CHECK-NEXT:    [[C3:%.*]] = icmp uge i32 [[UMAX2]], [[A]]
-; CHECK-NEXT:    [[UMAX_OR_EQ3:%.*]] = select i1 [[C3]], i32 [[UMAX2]], i32 [[A]]
-; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX_OR_EQ1]], [[UMAX_OR_EQ3]]
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ugt i32 [[UMAX_OR_EQ1]], [[C:%.*]]
+; CHECK-NEXT:    [[UMAX_OR_EQ3_NARY:%.*]] = select i1 [[TMP1]], i32 [[UMAX_OR_EQ1]], i32 [[C]]
+; CHECK-NEXT:    [[RES:%.*]] = add i32 [[UMAX_OR_EQ1]], [[UMAX_OR_EQ3_NARY]]
 ; CHECK-NEXT:    ret i32 [[RES]]
 ;
   %c1 = icmp uge i32 %a, %b