diff --git a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h index 5aac7c2ac5d3e..f9fcd17662f90 100644 --- a/llvm/include/llvm/Transforms/Utils/SCCPSolver.h +++ b/llvm/include/llvm/Transforms/Utils/SCCPSolver.h @@ -194,10 +194,16 @@ class SCCPSolver { LLVM_ABI void visit(Instruction *I); LLVM_ABI void visitCall(CallInst &I); + /// Simplify instructions in \p BB using the solver's lattice information. + /// When \p Eager is true, also apply more aggressive folds that may rewrite + /// IR into forms less friendly to earlier canonicalization passes. Keep eager + /// mode for later optimization points where exposing extra range-based folds + /// outweighs the risk of hiding canonical patterns. LLVM_ABI bool simplifyInstsInBlock(BasicBlock &BB, SmallPtrSetImpl &InsertedValues, Statistic &InstRemovedStat, - Statistic &InstReplacedStat); + Statistic &InstReplacedStat, + bool Eager = false); LLVM_ABI bool removeNonFeasibleEdges(BasicBlock *BB, DomTreeUpdater &DTU, BasicBlock *&NewUnreachableBB) const; diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp index feee794ffeae1..eada8efddd8ff 100644 --- a/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -102,7 +102,8 @@ static bool runSCCP(Function &F, const DataLayout &DL, } MadeChanges |= Solver.simplifyInstsInBlock(BB, InsertedValues, - NumInstRemoved, NumInstReplaced); + NumInstRemoved, NumInstReplaced, + /*Eager=*/true); } // Remove unreachable blocks and non-feasible edges. diff --git a/llvm/lib/Transforms/Utils/SCCPSolver.cpp b/llvm/lib/Transforms/Utils/SCCPSolver.cpp index fd315c14df866..361fec88a60fd 100644 --- a/llvm/lib/Transforms/Utils/SCCPSolver.cpp +++ b/llvm/lib/Transforms/Utils/SCCPSolver.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/SCCPSolver.h" +#include "llvm/ADT/APInt.h" #include "llvm/ADT/SetVector.h" #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/InstructionSimplify.h" @@ -29,9 +30,9 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/Local.h" #include +#include #include #include @@ -105,6 +106,412 @@ static ConstantRange getRange(Value *Op, SCCPSolver &Solver, /*UndefAllowed=*/false); } +namespace { + +// Sibling of ConstantRange::getNonEmptyRange +ConstantRange getMightEmptyRange(const APInt &L, const APInt &R) { + return L == R ? ConstantRange::getEmpty(L.getBitWidth()) + : ConstantRange(L, R); +} + +/// Represents periodic mapping f(x) = Cx mod M +class ModularMulMapping { +public: + ModularMulMapping(const APInt &C, const APInt &M) + : MulC(C), Modulus(M), IsFullMod(M.isZero()), IsURem(C.isOne()), + WideBits(C.getBitWidth() * 2), WideMulC(C.zext(WideBits)), + WideModulus(M.zext(WideBits)) { + assert((C.isStrictlyPositive() || C.isMinSignedValue()) && + "Expected a positive multiplier C"); + assert(!IsURem || !M.isZero() && "Expected a valid modulus M if C = 1"); + } + + // Mapping: f(x) = Cx mod M = ((C mod M) * (x mod M)) mod M + APInt operator()(const APInt &x) const { + assert(x.getBitWidth() == MulC.getBitWidth() && + "The bit width of x and C should be equal for f(x) = Cx"); + // Fast path for f(x) = Cx + if (IsFullMod) + return MulC * x; + // Fast path for f(x) = x mod M + if (IsURem) + return x.urem(Modulus); + return (WideMulC * x.zext(WideBits)) + .urem(WideModulus) + .trunc(MulC.getBitWidth()); + } + + ConstantRange getInvertibleImage(const ConstantRange &SrcCR, + bool &IsDomainReturned) const { + assert(!SrcCR.isEmptySet() && "Expected non-empty SrcCR"); + return SrcCR.isWrappedSet() + ? getInvertibleImageOnWrappedX(SrcCR, IsDomainReturned) + : getInvertibleImageOnUnwrappedX(SrcCR, IsDomainReturned); + } + +private: + ConstantRange getInvertibleImageOnUnwrappedX(const ConstantRange &SrcCR, + bool &IsDomainReturned) const { + assert(!SrcCR.isWrappedSet() && "Expected nuw SrcCR"); + const unsigned BW = MulC.getBitWidth(); + + const APInt &XLo = + SrcCR.isFullSet() ? APInt::getZero(BW) : SrcCR.getLower(), + &XHi = + SrcCR.isFullSet() ? APInt::getZero(BW) : SrcCR.getUpper(); + // [Lo, Hi) = [Lo, Hi-1] + const APInt RangeSize = XHi - XLo - 1; + + // Periods k = floor(|Range| / T) + // = floor((SrcCR.size() - 1) / (M / C)) + // = floor((SrcCR.size() - 1) * C / M) + const APInt SizeMulC = RangeSize.zext(WideBits) * WideMulC; + const APInt Periods = + IsFullMod ? SizeMulC.lshr(BW) : SizeMulC.udiv(WideModulus); + + if (Periods.isZero()) { + // k = 0: the walk never wraps, so the mapping is invertible over the full + // result space. Here we return the reachable image/domain + // Y = f(CR) = [f(Lo), f(Hi)). + // Consider discreteness, Y = [ f(Lo) , f(Hi - 1) + 1 ) + // f(x) = Cx: Y | / + // Y | / + // | + // Y | / + // └----------- + // XXX + IsDomainReturned = true; + // SrcCR is not emptyset -> Y cannot be emptyset but might be fullset. + // f(Hi - 1) must ∈ [0, M), thus, we don't use modular addition to get the + // open right end f(Hi - 1) + 1. + return ConstantRange::getNonEmpty((*this)(XLo), (*this)(XHi - 1) + 1); + } + + if (Periods.isOne()) { + // k = 1: only the unique part invertible. + // As f(x) walks along f(Lo) --> ⊤ --> ⊥ --> f(Hi), + // the repeated part is [f(Lo), f(Hi)) and the invertiable part + // Y = [ f(Hi), f(Lo) ) + // Consider discreteness, Y = [ f(Hi - 1) + 1 , f(Lo) ) + // f(x) = Cx: Y | / + // Y | / + // | / / + // Y | / + // └----------- + // ~XXX~ + IsDomainReturned = false; + // [f(Lo), f(Hi)) repeated -> Y cannot be fullset but might be emptyset. + // We must use modular addition to get the closed left end f(Hi - 1) + 1. + return getMightEmptyRange(modularAdd((*this)(XHi - 1), 1), (*this)(XLo)); + } + // k >= 2: the walk overlaps itself too much to have a unique inverse. + // f(x) = Cx: | / / + // | / / + // Y does not exist. | / / / + // | / / + // └------------- + return ConstantRange::getEmpty(BW); + } + /// If the range of x is wrapped, the linearity of `f(x) = Cx mod M` breaks on + /// the end. E.g., as follows, the invertibe image Y is continuos, but the + /// related pre-image X is not. + /// f(x) = Cx: Y| / + /// | / / + /// | / / + /// Y|/ + /// └---------- + /// X~~X ~~ + ConstantRange getInvertibleImageOnWrappedX(const ConstantRange &SrcCR, + bool &IsDomainReturned) const { + assert(SrcCR.isWrappedSet() && "Expected wrapped SrcCR"); + // FIXME: support wrapped SrcCR. + return ConstantRange::getEmpty(MulC.getBitWidth()); + } + APInt modularAdd(const APInt &LHS, int RHS) const { + return Modulus.isZero() ? LHS + RHS : (RHS + LHS).urem(Modulus); + }; + + const APInt &MulC; + const APInt &Modulus; + const bool IsFullMod; + const bool IsURem; + const unsigned WideBits; + const APInt WideMulC; + const APInt WideModulus; +}; +} // namespace + +/// Refer to https://github.com/llvm/llvm-project/pull/186347 for the +/// underlying math model. +/// +/// Given a result constraint CR on y = f(x) = Step * x mod Modulus and a +/// source domain X = SrcCR, try to compute a single interval CR' = f^{-1}(CR) +/// on x. +/// +/// We first compute the invertible result interval Y for x \in SrcCR, then try +/// to apply f^{-1} on CR or CR.inverse(). This is valid iff CR ⊆ Y, or iff CR +/// intersects the reachable image when Y itself is the domain. Modulus == 0 +/// denotes the full iBW ring, i.e. mod 2^BW. +static std::optional +getPreImageOfModularMul(const ConstantRange &CmpCR, const ConstantRange &SrcCR, + const APInt &C, const APInt &Modulus) { + assert(!C.isZero() && "Expected a non-zero periodic coefficient"); + assert(!CmpCR.isEmptySet() && "Unexpected empty constraint set"); + assert(!CmpCR.isFullSet() && "Unexpected full constraint set"); + assert(!SrcCR.isEmptySet() && "Unexpected empty input set"); + + const unsigned BW = C.getBitWidth(); + + ConstantRange Domain = + ConstantRange::getNonEmpty(APInt::getZero(BW), Modulus); + + const bool IsNegMulC = C.isMinSignedValue() + ? /* Treat smin as positive */ false + : C.isNegative(); + auto NegateRange = [](const ConstantRange &CR) -> ConstantRange { + // negate([L,R)) = - [L, R) = [1 - R, 1 - L) + return ConstantRange::getNonEmpty(1 - CR.getUpper(), 1 - CR.getLower()); + }; + // y = Cx \in CR --> -y = -Cx \in negate(CR) + const ConstantRange &ActiveCmpCR = IsNegMulC ? NegateRange(CmpCR) : CmpCR; + const APInt &Step = IsNegMulC ? -C : C; + + const ModularMulMapping Mapping{Step, Modulus}; + + // ==================================================================== // + // 1. Calculate the invertible interval Y for f(x) = Cx mod M. + // ==================================================================== // + + bool IsDomainReturned = false; + const ConstantRange Y = Mapping.getInvertibleImage(SrcCR, IsDomainReturned); + + if (Y.isEmptySet()) + return std::nullopt; + + if (IsDomainReturned) + Domain = Y; + + // ==================================================================== // + // 2. Calculate the equivalent range X via f^{-1} on CmpCR. + // ==================================================================== // + auto ModularSub = [&Modulus](const APInt &LHS, const APInt &RHS) { + return (Modulus.isZero() || LHS.uge(RHS)) ? LHS - RHS : Modulus - RHS + LHS; + }; + // Try to map CmpRange to its pre-image, i.e., f^{-1}(CR). + auto TryGetPreImage = + [&](const ConstantRange &CR) -> std::optional { + if (CR.contains(Domain)) + return /* Domain ⊆ CR*/ ConstantRange::getFull(BW); + if (CR.inverse().contains(Domain)) + return /* Domain ∩ CR = ∅ */ ConstantRange::getEmpty(BW); + + // ActiveCR is the reachable part of CR. + // ActiveCmpY = null if + // L-------U : Domain or --U L----- : Domain + // --U L----- : CR L-------U : CR + std::optional ActiveCR = Domain.exactIntersectWith(CR); + // If ActiveCmpY = null or ActiveCR ⊈ the invertible image Y, + // there are >1 separate intervals of x, making Cx ∈ CR. + // I.e., we cannot derive a single X. + if (!ActiveCR || !Y.contains(*ActiveCR)) + return std::nullopt; + + // ActiveCR.Hi is not belong to Y, thus we use Y1 = ActiveCR.Hi - 1 + const APInt &Y0 = ActiveCR->getLower(), + &Y1 = ModularSub(ActiveCR->getUpper(), APInt(BW, 1)); + // Fast path for Lo = 0: X = [ y0 / C , y1 / C ) + if (SrcCR.getLower().isZero()) { + // f(x) = Cx: Y | / X must be Y / C without mod directly. + // Y | / + // | / / + // |/ / + // └--------- + // ~~XX~~ + const APInt X0 = APIntOps::RoundingUDiv(Y0, Step, APInt::Rounding::UP), + X1 = APIntOps::RoundingUDiv(Y1, Step, APInt::Rounding::DOWN) + + 1; + return getMightEmptyRange(X0, X1); + } + + // Given SrcCR = [Lo, Hi) and invertible interval CR = [y0, y1], + // we need to find X = [x0, x1] ⊆ SrcCR, s.t., f(X) = CR. + // I.e., y0 = f(x0) + // DeltaY = y0 - f(Lo) = f(x0) - f(Lo) = f(x0 - Lo) = C * DeltaX + // DeltaX = DeltaY / C = (y0 - f(Lo)) / C + // x0 = Lo + DeltaX + // As y0, f(Lo) ∈ [0, M), we do need to consider modulus. + // x1 shares the same derivation. + // Considering discreteness, we need to adjust X = [A,B) properly as + // follows. + // X = [ceil(x0), ceil(x1)) + const APInt LoY = Mapping(SrcCR.getLower()); + const APInt DeltaY0 = ModularSub(Y0, LoY), DeltaY1 = ModularSub(Y1, LoY); + const APInt DeltaX0 = + APIntOps::RoundingUDiv(DeltaY0, Step, APInt::Rounding::UP); + const APInt DeltaX1 = + APIntOps::RoundingUDiv(DeltaY1, Step, APInt::Rounding::DOWN); + const APInt X0 = SrcCR.getLower() + DeltaX0, + X1 = SrcCR.getLower() + DeltaX1 + 1; + const ConstantRange X = getMightEmptyRange(X0, X1); + assert(SrcCR.contains(X) && "X should be subset of SrcCR"); + return X; + }; + + // Try to get single X = f^{-1}(CmpCR) to make Cx ∈ CmpCR. + if (auto X = TryGetPreImage(ActiveCmpCR)) + return *X; + + // Try to get single X = f^{-1}(CmpCR.inverse()).inverse() to make Cx ∈ CmpCR. + if (auto X = TryGetPreImage(ActiveCmpCR.inverse())) + return X->inverse(); + + return std::nullopt; +} + +/// Given CmpCR constraining y = f(x) and SCCP's known range SrcCR for x, try to +/// rewrite the constraint as a single ConstantRange on x. +/// +/// This only handles mappings that the current solver models via +/// getPreImageOfModularMul(): +/// - mul x, C : y = C * x mod 2^BW +/// - shl x, C : y = (2^C) * x mod 2^BW +/// - urem x, C : y = x mod C +/// - and x, C : y = x mod C+1 if C is low-bit mask +/// +/// Returns nullopt if the reachable image from SrcCR does not admit one +/// invertible interval, or if the preimage of CmpCR cannot be expressed as one +/// ConstantRange. +static std::optional +getPreImageOfInvertiblePeriodicMapping(unsigned Opcode, const APInt &C, + const ConstantRange &SrcCR, + const ConstantRange &CmpCR) { + // TODO: Support srem and other more complex periodic mappings. + switch (Opcode) { + case Instruction::Mul: + // y = C*x = C*x mod (MAX + 1) + return getPreImageOfModularMul(CmpCR, SrcCR, C, APInt(C.getBitWidth(), 0)); + case Instruction::Shl: + // y = x << C = 2^C * x mod (MAX + 1) + return getPreImageOfModularMul( + CmpCR, SrcCR, APInt::getOneBitSet(C.getBitWidth(), C.getZExtValue()), + APInt(C.getBitWidth(), 0)); + case Instruction::And: + assert(C.isMask() && "Expected a low-bit mask C"); + // y = x & C = 1 * x mod C + 1 + return getPreImageOfModularMul(CmpCR, SrcCR, APInt(C.getBitWidth(), 1), + C + 1); + case Instruction::URem: + // y = x % C = 1 * x mod C + return getPreImageOfModularMul(CmpCR, SrcCR, APInt(C.getBitWidth(), 1), C); + default: + assert(false && "Unsupported invertible periodic linear mapping opcode"); + } + + return std::nullopt; +} + +/// SCCP already proves x \in KnownCR, so only ActiveCmpCR = CmpCR ∩ KnownCR +/// matters. Try to replace CmpCR with a simpler equivalent range NewCmpCR +/// such that NewCmpCR ∩ KnownCR == ActiveCmpCR. +/// +/// Prefer ranges that lower to a single canonical compare without an add: +/// - [L, L+1) --> X eq L +/// - [R+1, R) --> X ne R +/// - [0, R) --> X ult R +/// - [L, 0) --> X uge L +/// - [SignMin, R) --> X slt R +/// - [L, SignMin) --> X sge L +/// +/// If no such range preserves the active semantics under KnownCR, keep CmpCR. +static ConstantRange simplifyCmpRange(const ConstantRange &CmpCR, + const ConstantRange &KnownCR) { + assert(!KnownCR.inverse().contains(CmpCR) && + "CmpCR ∩ KnowCR should not be ∅"); + assert((!CmpCR.isFullSet() && !CmpCR.isEmptySet()) && "Unexpected CmpCR"); + assert((!KnownCR.isFullSet() && !KnownCR.isEmptySet()) && + "Unexpected KnownCR"); + + const unsigned BW = CmpCR.getBitWidth(); + // All reachable value satisfy CmpCR --> always true. + if (CmpCR.contains(KnownCR)) + return ConstantRange::getFull(BW); + + std::optional ActCmpCR = CmpCR.exactIntersectWith(KnownCR); + if (!ActCmpCR) + return CmpCR; + // Proof of ActCmpCR cannot be ne: + // 1. ActCmpCR = ne ∧ ActCmpCR ⊆ KnownCR -> KnownCR = ActCmpCR/fullset + // 2. KnownCR = fullset contradicts KnownCR != fullset + // 3. KnownCR = ActCmpCR = KnownCR ∩ CmpCR -> KnownCR ⊆ CmpCR + // 4. KnownCR ⊆ CmpCR contradicts KnownCR ⊈ CmpCR + assert(/*ne*/ !ActCmpCR->inverse().isSingleElement() && "Unexpected ne"); + + // We prefer eq rather than ne. + if (/*eq*/ ActCmpCR->isSingleElement()) + return *ActCmpCR; + + // We prefer ne rather than lt/ge. + // L--------R : KnownCR or L------------R : KnownCR + // L-------R : ActiveCmpCR L-----------R : ActiveCmpCR + // ---RL-------------- : RelaxedCmpCR ---------------RL-- : RelaxedCmpCR + if (const ConstantRange FalseCR = KnownCR.intersectWith(ActCmpCR->inverse()); + FalseCR.isSingleElement()) + return FalseCR.inverse(); + + const APInt &CmpLo = ActCmpCR->getLower(), &CmpHi = ActCmpCR->getUpper(); + + // If the intersection happens to be the ONE-icmp check, just return it. + if (/*ult*/ CmpLo.isZero() || + /*slt*/ CmpLo.isMinSignedValue() || + /*uge*/ CmpHi.isZero() || + /*sge*/ CmpHi.isMinSignedValue()) + return *ActCmpCR; + + const APInt Zero = APInt::getZero(BW); + const APInt SignMin = APInt::getSignedMinValue(BW); + + if (CmpLo == KnownCR.getLower()) { + // Tie to lower: + + // Try ult. + // 0 + // | L------------R : KnownCR + // | L---R : ActiveCmpCR + // L------R : RelaxedCmpCR + if (!KnownCR.isWrappedSet()) + return ConstantRange::getNonEmpty(Zero, CmpHi); + + // Try slt. + // smin smin + // -----R | L------- : KnownCR ----R | L------- : KnownCR + // | L--R : ActiveCmpCR --R | L------- : ActiveCmpCR + // L-----R : RelaxedCmpCR --R L---------- : RelaxedCmpCR + if (!KnownCR.isSignWrappedSet()) + return ConstantRange::getNonEmpty(SignMin, CmpHi); + + } else if (CmpHi == KnownCR.getUpper()) { + // Tie to upper: + + // Try uge. + // 0 + // | L--------R : KnownCR + // | L---R : ActiveCmpCR + // R L---------- : RelaxedCmpCR + if (!KnownCR.isWrappedSet()) + return ConstantRange::getNonEmpty(CmpLo, Zero); + + // Try sge. + // smin smin + // -----R | L------- : KnownCR -----R | L------- : KnownCR + // L--R | : ActiveCmpCR -----R | L--- : ActiveCmpCR + // L-----R : RelaxedCmpCR --------R L--- : RelaxedCmpCR + if (!KnownCR.isSignWrappedSet()) + return ConstantRange::getNonEmpty(CmpLo, SignMin); + } + + return CmpCR; +} + /// Try to use \p Inst's value range from \p Solver to infer the NUW flag. static bool refineInstruction(SCCPSolver &Solver, const SmallPtrSetImpl &InsertedValues, @@ -238,10 +645,86 @@ static bool replaceSignedInst(SCCPSolver &Solver, return true; } +/// Revert f(x) ∈ @p CmpCR to x ∈ R', where f(x) = op(x, C), x ∈ @p XRange. +static Value *revertMappingRangeCheck(Value *X, const unsigned OpCode, + const APInt &C, + const ConstantRange &CmpCR, + const ConstantRange &XRange, + SmallPtrSetImpl &InsertedValues, + ICmpInst &ICmpBeingReplaced) { + // We support integer vector/scalar. + // For vector, the mapping must be fixed, i.e., splat C. + assert(X->getType()->getScalarType()->isIntegerTy() && + "Only support integer mapping"); + auto XCmpCR = + getPreImageOfInvertiblePeriodicMapping(OpCode, C, XRange, CmpCR); + if (!XCmpCR) + return nullptr; + + IRBuilder Builder(&ICmpBeingReplaced); + if (XCmpCR->isEmptySet()) + return Builder.getFalse(); + if (XCmpCR->isFullSet()) + return Builder.getTrue(); + + // Use XRange to simplify XCmpCR. E.g.: + // XCmpCR = [5, 10), *XCmpCR = [5, 0) --> NewCmpCR = [0, 10) -> ult + *XCmpCR = simplifyCmpRange(*XCmpCR, XRange); + + // Emit XCmpCR as icmp Pred (X + C1), C2 + ICmpInst::Predicate Pred; + APInt RHS, Offset; + XCmpCR->getEquivalentICmp(Pred, RHS, Offset); + + if (!Offset.isZero()) { + X = Builder.CreateAdd(X, ConstantInt::get(X->getType(), Offset)); + InsertedValues.insert(X); + } + + Value *NewICmp = + Builder.CreateICmp(Pred, X, ConstantInt::get(X->getType(), RHS)); + InsertedValues.insert(NewICmp); + return NewICmp; +} + +/// Given X ∈ @p XRange, relax X ∈ @p CmpCR into single icmp [pred] X, C. +static Value *relaxTwoInstRangeCheck(Value *X, const ConstantRange &XRange, + const ConstantRange &CmpCR, + SmallPtrSetImpl &InsertedValues, + ICmpInst &ICmpBeingReplaced) { + // One ICmp range check : icmp pred X, C + // Bail out if CmpCR is already represented by ONE icmp. + if (X == ICmpBeingReplaced.getOperand(0)) + return nullptr; + + // Early exit if we know nothing about X. + if (XRange.isFullSet()) + return nullptr; + + // We are allowed to refine the comparison to either true or false for + // out of range inputs. Based on this, try to simplify CmpCR as a single + // ult/uge/slt/sge/eq/ne. + // E.g., CmpCR = [3, 10), YRange = [5, 0) --> NewCmpCR = [0, 10) -> ult + ConstantRange NewCmpCR = simplifyCmpRange(CmpCR, XRange); + + ICmpInst::Predicate Pred; + APInt RHS; + + // Fail to simplify CmpCR as single icmp. + if (!NewCmpCR.getEquivalentICmp(Pred, RHS)) + return nullptr; + + IRBuilder Builder(&ICmpBeingReplaced); + Value *NewICmp = + Builder.CreateICmp(Pred, X, ConstantInt::get(X->getType(), RHS)); + InsertedValues.insert(NewICmp); + return NewICmp; +} + /// Try to use \p Inst's value range from \p Solver to simplify it. static Value *simplifyInstruction(SCCPSolver &Solver, SmallPtrSetImpl &InsertedValues, - Instruction &Inst) { + Instruction &Inst, bool Eager) { auto GetRange = [&Solver, &InsertedValues](Value *Op) { return getRange(Op, Solver, InsertedValues); }; @@ -287,59 +770,65 @@ static Value *simplifyInstruction(SCCPSolver &Solver, return Sub; } - // Relax range checks. + // Check if we can relax icmp Pred, Y, ... to a simpler form. if (auto *ICmp = dyn_cast(&Inst)) { - Value *X; - auto MatchTwoInstructionExactRangeCheck = - [&]() -> std::optional { + Value *Y; + auto MatchExactRangeCheck = [&]() -> std::optional { const APInt *RHSC; + // Match icmp Pred LHS, C if (!match(ICmp->getOperand(1), m_APInt(RHSC))) return std::nullopt; Value *LHS = ICmp->getOperand(0); ICmpInst::Predicate Pred = ICmp->getPredicate(); const APInt *Offset; - if (match(LHS, m_OneUse(m_AddLike(m_Value(X), m_APInt(Offset))))) - return ConstantRange::makeExactICmpRegion(Pred, *RHSC).sub(*Offset); - // Match icmp eq/ne X & NegPow2, C + // FIXME: Relax this for reverting f(x) ∈ R to x ∈ R'? + if (!LHS->hasOneUse()) + return std::nullopt; + const ConstantRange ExactCmpCR = + ConstantRange::makeExactICmpRegion(Pred, *RHSC); + // Match icmp Pred Y + C1, C2 + if (match(LHS, m_AddLike(m_Value(Y), m_APInt(Offset)))) + return ExactCmpCR.sub(*Offset); + // Match icmp Pred Y - C1, C2 + if (match(LHS, m_Sub(m_Value(Y), m_APInt(Offset)))) + return ExactCmpCR.add(*Offset); + // Match icmp eq/ne Y & NegPow2, C if (ICmp->isEquality()) { const APInt *Mask; - if (match(LHS, m_OneUse(m_And(m_Value(X), m_NegatedPower2(Mask)))) && + if (match(LHS, m_And(m_Value(Y), m_NegatedPower2(Mask))) && RHSC->countr_zero() >= Mask->countr_zero()) { ConstantRange CR(*RHSC, *RHSC - *Mask); return Pred == ICmpInst::ICMP_EQ ? CR : CR.inverse(); } } - return std::nullopt; + Y = LHS; + return ExactCmpCR; }; - if (auto CR = MatchTwoInstructionExactRangeCheck()) { - ConstantRange LRange = GetRange(X); - // Early exit if we know nothing about X. - if (LRange.isFullSet()) - return nullptr; - auto ConvertCRToICmp = - [&](const std::optional &NewCR) -> Value * { - ICmpInst::Predicate Pred; - APInt RHS; - // Check if we can represent NewCR as an icmp predicate. - if (NewCR && NewCR->getEquivalentICmp(Pred, RHS)) { - IRBuilder Builder(&Inst); - Value *NewICmp = - Builder.CreateICmp(Pred, X, ConstantInt::get(X->getType(), RHS)); - InsertedValues.insert(NewICmp); - return NewICmp; - } - return nullptr; - }; - // We are allowed to refine the comparison to either true or false for out - // of range inputs. - // Here we refine the comparison to false, and check if we can narrow the - // range check to a simpler test. - if (auto *V = ConvertCRToICmp(CR->exactIntersectWith(LRange))) - return V; - // Here we refine the comparison to true, i.e. we relax the range check. - if (auto *V = ConvertCRToICmp(CR->exactUnionWith(LRange.inverse()))) + // Match icmp Pred, (op Y, C1), C2 as Y ∈ CmpCR. + if (auto CmpCR = MatchExactRangeCheck()) { + + // TODO: support more mappings f + // FIXME: should we treat trunc as x % 2^N? + // In eager mode, try to simplify Y = f(X) ∈ CR into X ∈ CR'. This is a + // more aggressive rewrite that can expose additional SCCP opportunities, + // but may also hide canonical forms expected by earlier passes. + if (const APInt *C; + Eager && /* the sole use of y = f(x) is icmp */ Y->hasOneUse() && + (match(Y, m_c_Mul(m_Value(X), m_APInt(C))) || + match(Y, m_Shl(m_Value(X), m_APInt(C))) || + match(Y, m_URem(m_Value(X), m_APInt(C))) || + match(Y, m_And(m_Value(X), m_LowBitMask(C))))) { + if (Value *V = revertMappingRangeCheck( + X, cast(Y)->getOpcode(), *C, *CmpCR, GetRange(X), + InsertedValues, *ICmp)) + return V; + } + + // Given Y ∈ YRange, try to simplify Y ∈ CR as single icmp. + if (Value *V = relaxTwoInstRangeCheck(Y, GetRange(Y), *CmpCR, + InsertedValues, *ICmp)) return V; } } @@ -350,7 +839,7 @@ static Value *simplifyInstruction(SCCPSolver &Solver, bool SCCPSolver::simplifyInstsInBlock(BasicBlock &BB, SmallPtrSetImpl &InsertedValues, Statistic &InstRemovedStat, - Statistic &InstReplacedStat) { + Statistic &InstReplacedStat, bool Eager) { bool MadeChanges = false; for (Instruction &Inst : make_early_inc_range(BB)) { if (Inst.getType()->isVoidTy()) @@ -366,7 +855,8 @@ bool SCCPSolver::simplifyInstsInBlock(BasicBlock &BB, ++InstReplacedStat; } else if (refineInstruction(*this, InsertedValues, Inst)) { MadeChanges = true; - } else if (auto *V = simplifyInstruction(*this, InsertedValues, Inst)) { + } else if (auto *V = + simplifyInstruction(*this, InsertedValues, Inst, Eager)) { Inst.replaceAllUsesWith(V); Inst.eraseFromParent(); ++InstRemovedStat; diff --git a/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll b/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll index 04eae7d2941d8..72d7651509f9e 100644 --- a/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll +++ b/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll @@ -111,8 +111,7 @@ define i32 @PR56119(i32 %e.coerce) { ; O1-LABEL: @PR56119( ; O1-NEXT: entry: ; O1-NEXT: [[CONV2:%.*]] = and i32 [[E_COERCE:%.*]], 255 -; O1-NEXT: [[REM:%.*]] = urem i32 [[CONV2]], 255 -; O1-NEXT: [[CMP:%.*]] = icmp eq i32 [[REM]], 7 +; O1-NEXT: [[CMP:%.*]] = icmp eq i32 [[CONV2]], 7 ; O1-NEXT: br i1 [[CMP]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] ; O1: if.then: ; O1-NEXT: tail call void (...) @foo() diff --git a/llvm/test/Transforms/SCCP/eager-invertible-periodic-mapping.ll b/llvm/test/Transforms/SCCP/eager-invertible-periodic-mapping.ll new file mode 100644 index 0000000000000..170659ad43c5b --- /dev/null +++ b/llvm/test/Transforms/SCCP/eager-invertible-periodic-mapping.ll @@ -0,0 +1,21 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes=sccp -S | FileCheck %s --check-prefix=SCCP +; RUN: opt < %s -passes=ipsccp -S | FileCheck %s --check-prefix=IPSCCP + +define i1 @mul_preimage_only_in_late_sccp(i8 range(i8 0, 5) %x) { +; SCCP-LABEL: define i1 @mul_preimage_only_in_late_sccp( +; SCCP-SAME: i8 range(i8 0, 5) [[X:%.*]]) { +; SCCP-NEXT: [[M:%.*]] = mul nuw nsw i8 [[X]], 17 +; SCCP-NEXT: [[CMP:%.*]] = icmp eq i8 [[X]], 0 +; SCCP-NEXT: ret i1 [[CMP]] +; +; IPSCCP-LABEL: define i1 @mul_preimage_only_in_late_sccp( +; IPSCCP-SAME: i8 range(i8 0, 5) [[X:%.*]]) { +; IPSCCP-NEXT: [[M:%.*]] = mul nuw nsw i8 [[X]], 17 +; IPSCCP-NEXT: [[CMP:%.*]] = icmp slt i8 [[M]], 17 +; IPSCCP-NEXT: ret i1 [[CMP]] +; + %m = mul i8 %x, 17 + %cmp = icmp slt i8 %m, 17 + ret i1 %cmp +} diff --git a/llvm/test/Transforms/SCCP/invertible-periodic-linear-mapping.ll b/llvm/test/Transforms/SCCP/invertible-periodic-linear-mapping.ll new file mode 100644 index 0000000000000..fb176b397c520 --- /dev/null +++ b/llvm/test/Transforms/SCCP/invertible-periodic-linear-mapping.ll @@ -0,0 +1,565 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt < %s -passes=sccp,dce -S | FileCheck %s + +; Test for icmp (mul (zext x), C) to icmp x, +; if mul is invertible on given predicate constraint +; Refer to https://github.com/llvm/llvm-project/pull/186347 to understand +; the mathematical model. + +; Comes from https://github.com/llvm/llvm-project/pull/185907#discussion_r2919506475 +; N = 9, M = 27 +; n = 2^9 = 512, m = 2^27 = 134217728, C = 262657 +; k = floor((n - 1) * C / m) = floor(511 * 262657 / 134217728) = 0 +; CR = [-2^26, 262657), Y = [0, 134217728) +; Invertible: yes +define i1 @slt_invertible_zext_mul_full_image(<2 x i9> %v) { +; CHECK-LABEL: define i1 @slt_invertible_zext_mul_full_image( +; CHECK-SAME: <2 x i9> [[V:%.*]]) { +; CHECK-NEXT: [[E:%.*]] = extractelement <2 x i9> [[V]], i64 0 +; CHECK-NEXT: [[Z:%.*]] = zext i9 [[E]] to i27 +; CHECK-NEXT: [[TMP1:%.*]] = add i27 [[Z]], -256 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i27 [[TMP1]], -255 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %e = extractelement <2 x i9> %v, i64 0 + %z = zext i9 %e to i27 + %m = mul i27 %z, 262657 + %cmp = icmp slt i27 %m, 262657 + ret i1 %cmp +} + +; N = 8, M = 16 +; n = 2^8 = 256, m = 2^16 = 65536, C = 257 +; k = floor((n - 1) * C / m) = floor(255 * 257 / 65536) = 0 +; CR = [-2^15, 257), Y = [0, 65536) +; Invertible: yes +define i1 @slt_invertible_zext_mul_full_image_i16(i8 %v) { +; CHECK-LABEL: define i1 @slt_invertible_zext_mul_full_image_i16( +; CHECK-SAME: i8 [[V:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i8 [[V]] to i16 +; CHECK-NEXT: [[TMP1:%.*]] = add i16 [[Z]], -128 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i16 [[TMP1]], -127 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %z = zext i8 %v to i16 + %m = mul nuw i16 %z, 257 + %cmp = icmp slt i16 %m, 257 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 17 +; k = floor((n - 1) * C / m) = floor(15 * 17 / 256) = 0 +; CR = [-2^7, 17), Y = [0, 256) +; Invertible: yes +define i1 @slt_invertible_zext_mul_full_image_i8(i4 %v) { +; CHECK-LABEL: define i1 @slt_invertible_zext_mul_full_image_i8( +; CHECK-SAME: i4 [[V:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[V]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -8 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], -7 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %z = zext i4 %v to i8 + %m = mul nuw i8 %z, 17 + %cmp = icmp slt i8 %m, 17 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 10 +; k = floor((n - 1) * C / m) = floor(15 * 10 / 256) = 0 +; CR = [0, 50), Y = [0, 151) +; Invertible: yes, because CR ⊆ Y +define i1 @ult_invertible_zext_mul_partial_image(i4 %x) { +; CHECK-LABEL: define i1 @ult_invertible_zext_mul_partial_image( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i8 [[Z]], 5 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 10 + %cmp = icmp ult i8 %m, 50 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 10 +; k = floor((n - 1) * C / m) = floor(15 * 10 / 256) = 0 +; CR = [0, 200), Y = [0, 151) +; Invertible: yes +define i1 @ult_invertible_zext_mul_all_true(i4 %x) { +; CHECK-LABEL: define i1 @ult_invertible_zext_mul_all_true( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: ret i1 true +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 10 + %cmp = icmp ult i8 %m, 200 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 10 +; k = floor((n - 1) * C / m) = floor(15 * 10 / 256) = 0 +; CR = [200, 256), Y = [0, 151) +; Invertible: yes +define i1 @uge_invertible_zext_mul_all_false(i4 %x) { +; CHECK-LABEL: define i1 @uge_invertible_zext_mul_all_false( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: ret i1 false +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 10 + %cmp = icmp uge i8 %m, 200 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CR = [60, 128), Y = [45, 256) +; Invertible: yes +define i1 @sge_invertible_tail_of_zext_mul(i4 %x) { +; CHECK-LABEL: define i1 @sge_invertible_tail_of_zext_mul( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -3 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 4 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 20 + %cmp = icmp sge i8 %m, 60 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CR = [60, 256), Y = [45, 256) +; Invertible: yes +define i1 @uge_invertible_tail_of_zext_mul(i4 %x) { +; CHECK-LABEL: define i1 @uge_invertible_tail_of_zext_mul( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -3 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 10 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 20 + %cmp = icmp uge i8 %m, 60 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 18 +; k = floor((n - 1) * C / m) = floor(15 * 18 / 256) = 1 +; CR = [-2^7, 16), Y = [15, 256) +; Invertible: yes on Y.inverse() = [16, 2^7) +define i1 @slt_noninvertible_signed_range_before_tail(i4 %v) { +; CHECK-LABEL: define i1 @slt_noninvertible_signed_range_before_tail( +; CHECK-SAME: i4 [[V:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[V]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -8 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], -7 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i4 %v to i8 + %cast = mul nuw i8 %z, 18 + %cmp = icmp slt i8 %cast, 16 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CR = [0, 45), Y = [45, 256) +; Invertible: yes on Y.inverse() = [45, 256) +define i1 @ult_noninvertible_zext_mul_range(i4 %x) { +; CHECK-LABEL: define i1 @ult_noninvertible_zext_mul_range( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -13 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], -10 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 20 + %cmp = icmp ult i8 %m, 45 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 18 +; k = floor((n - 1) * C / m) = floor(15 * 18 / 256) = 1 +; CR = [0, 16), Y = [15, 256) +; Invertible: yes on Y.inverse() = [16, 256) +define i1 @ult_noninvertible_zext_mul_before_tail(i4 %v) { +; CHECK-LABEL: define i1 @ult_noninvertible_zext_mul_before_tail( +; CHECK-SAME: i4 [[V:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[V]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -15 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], -14 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i4 %v to i8 + %cast = mul i8 %z, 18 + %cmp = icmp ult i8 %cast, 16 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CR = [-2^7, 60), Y = [45, 256) +; Invertible: yes on Y.inverse() = [60, 2^7) +define i1 @slt_noninvertible_crosses_wrap(i4 %v) { +; CHECK-LABEL: define i1 @slt_noninvertible_crosses_wrap( +; CHECK-SAME: i4 [[V:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[V]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -7 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], -4 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i4 %v to i8 + %cast = mul i8 %z, 20 + %cmp = icmp slt i8 %cast, 60 + ret i1 %cmp +} + +; Negative test +; N = 5, M = 8 +; n = 2^5 = 32, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(31 * 20 / 256) = 2 +; CR = [60, 256), Y = none +; Invertible: no +define i1 @uge_noninvertible_multiple_wraps(i5 %x) { +; CHECK-LABEL: define i1 @uge_noninvertible_multiple_wraps( +; CHECK-SAME: i5 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i5 [[X]] to i8 +; CHECK-NEXT: [[M:%.*]] = mul i8 [[Z]], 20 +; CHECK-NEXT: [[CMP:%.*]] = icmp uge i8 [[M]], 60 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i5 %x to i8 + %m = mul i8 %z, 20 + %cmp = icmp uge i8 %m, 60 + ret i1 %cmp +} + +; Tests for CmpCR built through add. + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CmpCR = [110, 228), Y = [45, 256) +; Invertible: yes +define i1 @sge_invertible_tail_of_zext_mul_plus_offset(i4 %x) { +; CHECK-LABEL: define i1 @sge_invertible_tail_of_zext_mul_plus_offset( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -6 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], 6 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 20 + %a = sub i8 %m, 100 + %cmp = icmp sge i8 %a, 10 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CmpCR = [100, 200), Y = [45, 256) +; Invertible: yes +define i1 @ult_invertible_zext_mul_plus_offset(i4 %x) { +; CHECK-LABEL: define i1 @ult_invertible_zext_mul_plus_offset( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -5 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 5 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 20 + %a = add i8 %m, -100 + %cmp = icmp ult i8 %a, 100 + ret i1 %cmp +} + +; N = 4, M = 8 +; n = 2^4 = 16, m = 2^8 = 256, C = 20 +; k = floor((n - 1) * C / m) = floor(15 * 20 / 256) = 1 +; CmpCR = [122, 60), Y = [45, 256) +; Invertible: yes on CmpCR.inverse() = [60, 122) +define i1 @slt_inverse_invertible_zext_mul_plus_offset(i4 %x) { +; CHECK-LABEL: define i1 @slt_inverse_invertible_zext_mul_plus_offset( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[Z:%.*]] = zext i4 [[X]] to i8 +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[Z]], -7 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], -4 +; CHECK-NEXT: ret i1 [[CMP]] +; + %z = zext i4 %x to i8 + %m = mul i8 %z, 20 + %a = add i8 %m, 6 + %cmp = icmp slt i8 %a, 66 + ret i1 %cmp +} + + +; TODO: support sext +; Test for icmp (mul (sext x), C) to icmp x. + +; Use plain i4 -> i8 sext instead of extra range metadata so the tests cover +; the extension pattern directly. +define i1 @sccp_sext_mul_shrinks_to_prefix(i4 %x) { +; CHECK-LABEL: define i1 @sccp_sext_mul_shrinks_to_prefix( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[S:%.*]] = sext i4 [[X]] to i8 +; CHECK-NEXT: [[M:%.*]] = mul i8 [[S]], 20 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i8 [[M]], 60 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %s = sext i4 %x to i8 + %m = mul i8 %s, 20 + %cmp = icmp ult i8 %m, 60 + ret i1 %cmp +} + + +define i1 @sccp_sext_mul_shrinks_to_negative_suffix(i4 %x) { +; CHECK-LABEL: define i1 @sccp_sext_mul_shrinks_to_negative_suffix( +; CHECK-SAME: i4 [[X:%.*]]) { +; CHECK-NEXT: [[S:%.*]] = sext i4 [[X]] to i8 +; CHECK-NEXT: [[M:%.*]] = mul i8 [[S]], 20 +; CHECK-NEXT: [[TMP1:%.*]] = icmp uge i8 [[M]], -60 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %s = sext i4 %x to i8 + %m = mul i8 %s, 20 + %cmp = icmp uge i8 %m, 196 + ret i1 %cmp +} + + +; Test for icmp (f(x), C) to icmp x + +define i1 @sccp_mul_wraps_once_shrinks_to_middle_window(i8 range(i8 0, 18) %x) { +; CHECK-LABEL: define i1 @sccp_mul_wraps_once_shrinks_to_middle_window( +; CHECK-SAME: i8 range(i8 0, 18) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X]], -5 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 8 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %m = mul i8 %x, 20 + %cmp = icmp uge i8 %m, 100 + ret i1 %cmp +} + +define i1 @sccp_urem_wraps_once_shrinks_to_middle_window(i8 range(i8 8, 19) %x) { +; CHECK-LABEL: define i1 @sccp_urem_wraps_once_shrinks_to_middle_window( +; CHECK-SAME: i8 range(i8 8, 19) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X]], -10 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 3 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %r = urem i8 %x, 10 + %cmp = icmp ult i8 %r, 3 + ret i1 %cmp +} + +define i1 @sccp_negative_mul_wraps_once_shrinks_to_middle_window(i8 range(i8 3, 17) %x) { +; CHECK-LABEL: define i1 @sccp_negative_mul_wraps_once_shrinks_to_middle_window( +; CHECK-SAME: i8 range(i8 3, 17) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X]], -10 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 3 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %m = mul i8 %x, -20 + %cmp = icmp ult i8 %m, 60 + ret i1 %cmp +} + +define i1 @sccp_shl_wraps_once_shrinks_to_upper_window(i8 range(i8 5, 23) %x) { +; CHECK-LABEL: define i1 @sccp_shl_wraps_once_shrinks_to_upper_window( +; CHECK-SAME: i8 range(i8 5, 23) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X]], -10 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[TMP1]], 6 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %y = shl i8 %x, 4 + %cmp = icmp uge i8 %y, 160 + ret i1 %cmp +} + +define i1 @sccp_mul_wraps_once_shrinks_to_two_value_window1(i8 range(i8 2, 6) %x) { +; CHECK-LABEL: define i1 @sccp_mul_wraps_once_shrinks_to_two_value_window1( +; CHECK-SAME: i8 range(i8 2, 6) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add i8 [[X]], -3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[TMP1]], 2 +; CHECK-NEXT: ret i1 [[CMP]] +; + %m = mul i8 %x, 100 + %a = add i8 %m, -200 + %cmp = icmp uge i8 %a, 100 + ret i1 %cmp +} + +; TODO: support wrapped-range +define i1 @sccp_mul_wraps_once_shrinks_to_two_value_window2(i8 range(i8 -2, 2) %x) { +; CHECK-LABEL: define i1 @sccp_mul_wraps_once_shrinks_to_two_value_window2( +; CHECK-SAME: i8 range(i8 -2, 2) [[X:%.*]]) { +; CHECK-NEXT: [[M:%.*]] = mul i8 [[X]], 100 +; CHECK-NEXT: [[A:%.*]] = add i8 [[M]], 56 +; CHECK-NEXT: [[CMP:%.*]] = icmp uge i8 [[A]], 100 +; CHECK-NEXT: ret i1 [[CMP]] +; + %m = mul i8 %x, 100 + %a = add i8 %m, -200 + %cmp = icmp uge i8 %a, 100 + ret i1 %cmp +} + +define i1 @sccp_mul_wraps_once_shrinks_to_singleton(i8 range(i8 0, 18) %x) { +; CHECK-LABEL: define i1 @sccp_mul_wraps_once_shrinks_to_singleton( +; CHECK-SAME: i8 range(i8 0, 18) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[X]], 7 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %m = mul i8 %x, 20 + %cmp = icmp eq i8 %m, 140 + ret i1 %cmp +} + +; TODO: support wrapped-range +; Wrapped-range counterparts for the generic f(x) tests above. + +define i1 @sccp_mul_wraps_once_shrinks_to_wrapped_window(i8 range(i8 250, 6) %x) { +; CHECK-LABEL: define i1 @sccp_mul_wraps_once_shrinks_to_wrapped_window( +; CHECK-SAME: i8 range(i8 -6, 6) [[X:%.*]]) { +; CHECK-NEXT: [[M:%.*]] = mul nsw i8 [[X]], 20 +; CHECK-NEXT: [[TMP2:%.*]] = icmp uge i8 [[M]], 100 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %m = mul i8 %x, 20 + %cmp = icmp uge i8 %m, 100 + ret i1 %cmp +} + +define i1 @sccp_urem_wraps_once_shrinks_to_wrapped_window(i8 range(i8 250, 10) %x) { +; CHECK-LABEL: define i1 @sccp_urem_wraps_once_shrinks_to_wrapped_window( +; CHECK-SAME: i8 range(i8 -6, 10) [[X:%.*]]) { +; CHECK-NEXT: [[R:%.*]] = urem i8 [[X]], 10 +; CHECK-NEXT: [[TMP2:%.*]] = icmp uge i8 [[R]], 6 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %r = urem i8 %x, 10 + %cmp = icmp uge i8 %r, 6 + ret i1 %cmp +} + +define i1 @sccp_negative_mul_wraps_once_shrinks_to_wrapped_window(i8 range(i8 250, 6) %x) { +; CHECK-LABEL: define i1 @sccp_negative_mul_wraps_once_shrinks_to_wrapped_window( +; CHECK-SAME: i8 range(i8 -6, 6) [[X:%.*]]) { +; CHECK-NEXT: [[M:%.*]] = mul nsw i8 [[X]], -20 +; CHECK-NEXT: [[TMP2:%.*]] = icmp ult i8 [[M]], 60 +; CHECK-NEXT: ret i1 [[TMP2]] +; + %m = mul i8 %x, -20 + %cmp = icmp ult i8 %m, 60 + ret i1 %cmp +} + +define i1 @sccp_shl_wraps_once_shrinks_to_wrapped_suffix(i8 range(i8 250, 6) %x) { +; CHECK-LABEL: define i1 @sccp_shl_wraps_once_shrinks_to_wrapped_suffix( +; CHECK-SAME: i8 range(i8 -6, 6) [[X:%.*]]) { +; CHECK-NEXT: [[Y:%.*]] = shl nsw i8 [[X]], 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp uge i8 [[Y]], -96 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %y = shl i8 %x, 4 + %cmp = icmp uge i8 %y, 160 + ret i1 %cmp +} + +define i1 @sccp_mul_with_offset_wraps_once_shrinks_to_wrapped_window(i8 range(i8 250, 6) %x) { +; CHECK-LABEL: define i1 @sccp_mul_with_offset_wraps_once_shrinks_to_wrapped_window( +; CHECK-SAME: i8 range(i8 -6, 6) [[X:%.*]]) { +; CHECK-NEXT: [[M:%.*]] = mul nsw i8 [[X]], 20 +; CHECK-NEXT: [[A:%.*]] = add i8 [[M]], 56 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i8 [[A]], 37 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %m = mul i8 %x, 20 + %a = add i8 %m, -200 + %cmp = icmp ult i8 %a, 37 + ret i1 %cmp +} + +define i1 @sccp_mul_wraps_once_shrinks_to_wrapped_domain_singleton(i8 range(i8 252, 8) %x) { +; CHECK-LABEL: define i1 @sccp_mul_wraps_once_shrinks_to_wrapped_domain_singleton( +; CHECK-SAME: i8 range(i8 -4, 8) [[X:%.*]]) { +; CHECK-NEXT: [[M:%.*]] = mul i8 [[X]], 20 +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[M]], -116 +; CHECK-NEXT: ret i1 [[TMP1]] +; + %m = mul i8 %x, 20 + %cmp = icmp eq i8 %m, 140 + ret i1 %cmp +} + +; Test for vector : f(vec) = C * vec, C is a splat constant. + +define <4 x i1> @vec_splat_and(<4 x i32> range(i32 100, 456) %x) { +; CHECK-LABEL: define <4 x i1> @vec_splat_and( +; CHECK-SAME: <4 x i32> range(i32 100, 456) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X]], splat (i32 -230) +; CHECK-NEXT: [[CMP:%.*]] = icmp ult <4 x i32> [[TMP1]], splat (i32 26) +; CHECK-NEXT: ret <4 x i1> [[CMP]] +; + %m = and <4 x i32> %x, splat (i32 255) + %cmp = icmp uge <4 x i32> %m, splat (i32 230) + ret <4 x i1> %cmp +} + +define <4 x i1> @vec_splat_mul(<4 x i8> range(i8 10, 17) %x) { +; CHECK-LABEL: define <4 x i1> @vec_splat_mul( +; CHECK-SAME: <4 x i8> range(i8 10, 17) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i8> [[X]], splat (i8 -12) +; CHECK-NEXT: [[CMP:%.*]] = icmp ult <4 x i8> [[TMP1]], splat (i8 2) +; CHECK-NEXT: ret <4 x i1> [[CMP]] +; + %m = mul <4 x i8> %x, splat (i8 50) + %add = add <4 x i8> %m, splat (i8 -40) + %cmp = icmp ult <4 x i8> %add, splat (i8 110) + ret <4 x i1> %cmp +} + +define <4 x i1> @vec_splat_urem(<4 x i32> range(i32 1, 10) %x) { +; CHECK-LABEL: define <4 x i1> @vec_splat_urem( +; CHECK-SAME: <4 x i32> range(i32 1, 10) [[X:%.*]]) { +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[X]], splat (i32 -2) +; CHECK-NEXT: [[CMP:%.*]] = icmp ult <4 x i32> [[TMP1]], splat (i32 6) +; CHECK-NEXT: ret <4 x i1> [[CMP]] +; + %m = urem <4 x i32> %x, splat (i32 8) + %cmp = icmp uge <4 x i32> %m, splat (i32 2) + ret <4 x i1> %cmp +} + +define <4 x i1> @vec_splat_shl(<4 x i8> range(i8 1, 4) %x) { +; CHECK-LABEL: define <4 x i1> @vec_splat_shl( +; CHECK-SAME: <4 x i8> range(i8 1, 4) [[X:%.*]]) { +; CHECK-NEXT: [[CMP:%.*]] = icmp ne <4 x i8> [[X]], splat (i8 2) +; CHECK-NEXT: ret <4 x i1> [[CMP]] +; + %m = shl <4 x i8> %x, splat (i8 7) + %cmp = icmp uge <4 x i8> %m, splat (i8 100) + ret <4 x i1> %cmp +} diff --git a/llvm/test/Transforms/SCCP/relax-range-checks.ll b/llvm/test/Transforms/SCCP/relax-range-checks.ll index 34e48136df37a..998271b3b24d9 100644 --- a/llvm/test/Transforms/SCCP/relax-range-checks.ll +++ b/llvm/test/Transforms/SCCP/relax-range-checks.ll @@ -113,4 +113,127 @@ define i1 @range_check_to_icmp_eq2(i32 range(i32 -1, 2) %x) { ret i1 %cmp } +define i1 @range_check_to_icmp_ult(i8 range(i8 2, 10) %x) { +; CHECK-LABEL: define i1 @range_check_to_icmp_ult( +; CHECK-SAME: i8 range(i8 2, 10) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i8 [[X]], -2 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[X]], 6 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add i8 %x, -2 + %cmp = icmp ult i8 %off, 4 + ret i1 %cmp +} + +define i1 @range_check_to_icmp_uge(i8 range(i8 2, 6) %x) { +; CHECK-LABEL: define i1 @range_check_to_icmp_uge( +; CHECK-SAME: i8 range(i8 2, 6) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i8 [[X]], -4 +; CHECK-NEXT: [[CMP:%.*]] = icmp uge i8 [[X]], 4 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add nsw i8 %x, -4 + %cmp = icmp ult i8 %off, 2 + ret i1 %cmp +} + +define i1 @range_check_to_icmp_slt(i8 range(i8 -56, 20) %x) { +; CHECK-LABEL: define i1 @range_check_to_icmp_slt( +; CHECK-SAME: i8 range(i8 -56, 20) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i8 [[X]], 56 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i8 [[X]], -6 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add nsw i8 %x, 56 + %cmp = icmp ult i8 %off, 50 + ret i1 %cmp +} + +define i1 @range_check_to_icmp_sge(i8 range(i8 -56, 20) %x) { +; CHECK-LABEL: define i1 @range_check_to_icmp_sge( +; CHECK-SAME: i8 range(i8 -56, 20) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i8 [[X]], 16 +; CHECK-NEXT: [[CMP:%.*]] = icmp sge i8 [[X]], -16 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add nsw i8 %x, 16 + %cmp = icmp ult i8 %off, 36 + ret i1 %cmp +} + +; Cover the early exit when ActiveCmpCR is already a one-icmp check. + +define i1 @range_check_intersection_to_icmp_eq(i32 range(i32 0, 4) %x) { +; CHECK-LABEL: define i1 @range_check_intersection_to_icmp_eq( +; CHECK-SAME: i32 range(i32 0, 4) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i32 [[X]], -3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i32 [[X]], 3 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add nsw i32 %x, -3 + %cmp = icmp ult i32 %off, 2 + ret i1 %cmp +} + +define i1 @range_check_intersection_to_icmp_ult(i8 range(i8 0, 10) %x) { +; CHECK-LABEL: define i1 @range_check_intersection_to_icmp_ult( +; CHECK-SAME: i8 range(i8 0, 10) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nuw nsw i8 [[X]], 2 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[X]], 4 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add i8 %x, 2 + %cmp = icmp ult i8 %off, 6 + ret i1 %cmp +} + +define i1 @range_check_intersection_to_icmp_slt(i8 range(i8 -128, -100) %x) { +; CHECK-LABEL: define i1 @range_check_intersection_to_icmp_slt( +; CHECK-SAME: i8 range(i8 -128, -100) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add i8 [[X]], -120 +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i8 [[X]], -118 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add i8 %x, -120 + %cmp = icmp ult i8 %off, 18 + ret i1 %cmp +} + +define i1 @range_check_intersection_to_icmp_uge(i8 range(i8 -6, 0) %x) { +; CHECK-LABEL: define i1 @range_check_intersection_to_icmp_uge( +; CHECK-SAME: i8 range(i8 -6, 0) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i8 [[X]], 2 +; CHECK-NEXT: [[CMP:%.*]] = icmp uge i8 [[X]], -2 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add i8 %x, 2 + %cmp = icmp ult i8 %off, 6 + ret i1 %cmp +} + +define i1 @range_check_intersection_to_icmp_sge(i8 range(i8 120, -128) %x) { +; CHECK-LABEL: define i1 @range_check_intersection_to_icmp_sge( +; CHECK-SAME: i8 range(i8 120, -128) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add nsw i8 [[X]], -122 +; CHECK-NEXT: [[CMP:%.*]] = icmp sge i8 [[X]], 122 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add i8 %x, -122 + %cmp = icmp ult i8 %off, 14 + ret i1 %cmp +} + +; Negative test: CmpCR relaxation cannot perform when x's range is nuw and nsw. +define i1 @range_check_nsw_nuw(i8 range(i8 -20, -56) %x) { +; CHECK-LABEL: define i1 @range_check_nsw_nuw( +; CHECK-SAME: i8 range(i8 -20, -56) [[X:%.*]]) { +; CHECK-NEXT: [[OFF:%.*]] = add i8 [[X]], 20 +; CHECK-NEXT: [[CMP:%.*]] = icmp ult i8 [[OFF]], 14 +; CHECK-NEXT: ret i1 [[CMP]] +; + %off = add i8 %x, 20 + %cmp = icmp ult i8 %off, 14 + ret i1 %cmp +} + declare void @use(i8)