diff --git a/llvm/include/llvm/IR/ValueMap.h b/llvm/include/llvm/IR/ValueMap.h index 435347be3f3c1..1b9ade90293d8 100644 --- a/llvm/include/llvm/IR/ValueMap.h +++ b/llvm/include/llvm/IR/ValueMap.h @@ -169,6 +169,17 @@ class ValueMap { return I != Map.end() ? I->second : ValueT(); } + /// Return the entry for the specified key, or \p Default. This variant is + /// useful, because `lookup` cannot be used with non-default-constructible + /// values. + template > + ValueT lookup_or(const KeyT &Val, U &&Default) const { + typename MapT::const_iterator I = Map.find_as(Val); + if (I != Map.end()) + return I->second; + return std::forward(Default); + } + // Inserts key,value pair into the map if the key isn't already in the map. // If the key is already in the map, it returns false and doesn't update the // value. diff --git a/llvm/include/llvm/Transforms/Utils/LoopSplitUtils.h b/llvm/include/llvm/Transforms/Utils/LoopSplitUtils.h new file mode 100644 index 0000000000000..f01a8c5d74662 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/LoopSplitUtils.h @@ -0,0 +1,154 @@ +//===- LoopSplitUtils.h - Split a loop's iteration space --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Splits a counted loop's iteration space into a chain of per-partition +// sub-loops. See LoopSplitUtils.cpp for the structure produced. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_LOOPSPLITUTILS_H +#define LLVM_TRANSFORMS_UTILS_LOOPSPLITUTILS_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Transforms/Utils/ValueMapper.h" +#include + +namespace llvm { + +class DominatorTree; +class SCEV; +class SCEVExpander; +class ScalarEvolution; + +/// Splits a counted loop into a chain of per-partition sub-loops. +/// +/// Usage: +/// \code +/// LoopSplitUtils LSU(L, LI, SE, DT); +/// if (!LSU.isLegal()) +/// return false; +/// LSU.addPartition(S0, E0); // one call per partition, in order +/// LSU.addPartition(S1, E1); +/// LSU.split(); +/// \endcode +class LoopSplitUtils { +public: + LLVM_ABI LoopSplitUtils(Loop *L, LoopInfo *LI, ScalarEvolution *SE, + DominatorTree *DT) + : L(L), LI(LI), SE(SE), DT(DT) {} + + /// Analyze \p L and return true if it is a counted loop this utility can + /// split: a bottom-tested single-exit loop in LCSSA form with a unique + /// unit-step integer induction and a computable trip count. Must succeed + /// before split(). + LLVM_ABI bool isLegal(); + + /// Return the loop's induction variable. Valid only after isLegal() succeeds. + LLVM_ABI PHINode *getInductionVariable() const { + return L->getInductionVariable(*SE); + } + + /// Append an inclusive partition range [Start, End] in iteration order. + /// Partitions must tile the whole space: first Start = induction start, each + /// later Start = previous End +/- step, last End = induction end (desc: S >= + /// E). + /// + /// Bounds must be loop-invariant and representable in the induction type + /// without wrapping: a Start +/- offset that wraps past TYPE_MAX/MIN/0 looks + /// in-range and silently miscompiles. See LoopSplitUtils.cpp for the + /// rationale. + /// + /// Every partition is guarded by default; use avoidPartitionGuard() to opt + /// out. + LLVM_ABI void addPartition(const SCEV *Start, const SCEV *End); + + /// Suppress the entry guard for partition \p PartitionIndex (already added). + /// Use only for a partition the caller can prove runs at least once; for a + /// runtime- empty partition this is incorrect and yields one spurious + /// iteration. + LLVM_ABI void avoidPartitionGuard(unsigned PartitionIndex); + + LLVM_ABI unsigned getNumPartitions() const { return Partitions.size(); } + + /// Perform the split. Requires a successful isLegal() and at least two + /// partitions. Returns true if the loop was rewritten. + LLVM_ABI bool split(); + + /// Return the counterpart of original-loop value \p V in partition + /// \p PartitionIndex (0-based). Partition 0 maps values to themselves; a + /// later partition returns the clone, or null if not cloned. Valid only after + /// split(). + LLVM_ABI Value *getPartitionValue(Value *V, unsigned PartitionIndex) const; + + /// Return the original-to-clone value map for the partition at + /// \p PartitionIndex, for callers that want to remap many values. Null for + /// partition 0 (identity) and for any partition that was not cloned. + LLVM_ABI const ValueToValueMapTy * + getPartitionValueMap(unsigned PartitionIndex) const; + +private: + /// Everything known about one partition: the caller-supplied range plus the + /// state split() derives. Indexed by partition number in \c Partitions. + struct PartitionInfo { + PartitionInfo() = default; + PartitionInfo(const SCEV *StartExpr, const SCEV *EndExpr) + : StartExpr(StartExpr), EndExpr(EndExpr) {} + + // Set by addPartition() / avoidPartitionGuard() before split(): + const SCEV *StartExpr = nullptr; // inclusive iteration range [Start, End]. + const SCEV *EndExpr = nullptr; + bool Guarded = true; // emit an entry guard? + + // Filled in by split(): + std::unique_ptr VMap; // null for partition 0 (identity). + Value *StartVal = nullptr; // expanded start. + Value *SelEnd = nullptr; // clamped end min(End, indEnd). + bool Empty = false; // provably zero-iteration. + BasicBlock *GuardBlock = nullptr; + BasicBlock *Preheader = nullptr; + BasicBlock *Exit = nullptr; + Loop *SubLoop = nullptr; + Value *LatchIndOp = nullptr; // induction operand of the latch compare. + }; + + /// Per-split() scratch threaded through the phase helpers (the escaping + /// values, new blocks, etc.). A pure transform internal, so it is defined in + /// the implementation file. + struct SplitState; + + Loop *L; + LoopInfo *LI; + ScalarEvolution *SE; + DominatorTree *DT; + + // Induction analysis, populated by isLegal(). + Value *LatchIndOperand = nullptr; // induction operand of the latch compare. + bool InductionIsSigned = false; // iteration ordering signedness. + const SCEV *InductionEnd = nullptr; + + /// One record per partition, in add order. + SmallVector Partitions; + + // split() phase helpers, run in order; each is documented at its definition. + /// Collect loop-carried and live-out values and split off the final exit. + void collectEscapingValues(SplitState &S); + /// Expand each partition's start and clamped end into the entry guard. + void expandPartitionBounds(SplitState &S, SCEVExpander &Expander); + /// Pass 1: clone each later partition's sub-loop and create its guard/exit. + void clonePartitions(SplitState &S); + /// Pass 2: emit each guard, clamp each latch, and chain the partitions. + void chainPartitions(SplitState &S); + /// Rebuild SSA for every escaping value with a per-value SSAUpdater. + void reconstructSSA(SplitState &S); +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_LOOPSPLITUTILS_H diff --git a/llvm/include/llvm/Transforms/Utils/LoopSplitUtilsPass.h b/llvm/include/llvm/Transforms/Utils/LoopSplitUtilsPass.h new file mode 100644 index 0000000000000..4a744d5fb7a22 --- /dev/null +++ b/llvm/include/llvm/Transforms/Utils/LoopSplitUtilsPass.h @@ -0,0 +1,30 @@ +//===- LoopSplitUtilsPass.h - Test driver for LoopSplitUtils ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A command-line driven pass used to exercise the LoopSplitUtils utility from +// `opt`. The split points are provided via the -loop-split-points option as +// iteration offsets relative to the induction start. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_LOOPSPLITUTILSPASS_H +#define LLVM_TRANSFORMS_UTILS_LOOPSPLITUTILSPASS_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Support/Compiler.h" + +namespace llvm { + +class LoopSplitUtilsPass : public PassInfoMixin { +public: + LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_LOOPSPLITUTILSPASS_H diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index f758469dd8ebb..17d096eba7e36 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -380,6 +380,7 @@ #include "llvm/Transforms/Utils/InstructionNamer.h" #include "llvm/Transforms/Utils/LibCallsShrinkWrap.h" #include "llvm/Transforms/Utils/LoopSimplify.h" +#include "llvm/Transforms/Utils/LoopSplitUtilsPass.h" #include "llvm/Transforms/Utils/LoopVersioning.h" #include "llvm/Transforms/Utils/LowerCommentStringPass.h" #include "llvm/Transforms/Utils/LowerGlobalDtors.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 0955bfdb3246c..90593c1effa40 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -486,6 +486,7 @@ FUNCTION_PASS("loop-fusion", LoopFusePass()) FUNCTION_PASS("loop-load-elim", LoopLoadEliminationPass()) FUNCTION_PASS("loop-simplify", LoopSimplifyPass()) FUNCTION_PASS("loop-sink", LoopSinkPass()) +FUNCTION_PASS("loop-split-utils", LoopSplitUtilsPass()) FUNCTION_PASS("loop-versioning", LoopVersioningPass()) FUNCTION_PASS("lower-atomic", LowerAtomicPass()) FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass()) diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt index 68f48b757eb6d..8858582506329 100644 --- a/llvm/lib/Transforms/Utils/CMakeLists.txt +++ b/llvm/lib/Transforms/Utils/CMakeLists.txt @@ -49,6 +49,8 @@ add_llvm_component_library(LLVMTransformUtils LoopPeel.cpp LoopRotationUtils.cpp LoopSimplify.cpp + LoopSplitUtils.cpp + LoopSplitUtilsPass.cpp LoopUnroll.cpp LoopUnrollAndJam.cpp LoopUnrollRuntime.cpp diff --git a/llvm/lib/Transforms/Utils/LoopSplitUtils.cpp b/llvm/lib/Transforms/Utils/LoopSplitUtils.cpp new file mode 100644 index 0000000000000..1c8ea08ac80c6 --- /dev/null +++ b/llvm/lib/Transforms/Utils/LoopSplitUtils.cpp @@ -0,0 +1,617 @@ +//===- LoopSplitUtils.cpp - Split a loop's iteration space ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Splits a counted loop's iteration space into a chain of per-partition +// sub-loops. See LoopSplitUtils.h for the high-level usage guidelines. +// +// Structure produced for partitions [S0,E0], [S1,E1], ... where E is the loop's +// last iteration and each clamped end sel_i = min(E_i, E): +// +// guard0: ; every S_i and sel_i is computed here +// if (S0 <= sel0) goto preheader0 else goto guard1 ; default guard check +// loop0: ... ; latch stops at sel0 +// exit0 -> guard1 +// guard1: +// if (S1 <= sel1) goto preheader1 else goto guard2 ; default guard check +// loop1: ... ; latch stops at sel1 +// exit1 -> guard2 +// ... +// final.exit: ; merges every partition's live-outs +// +// Each guard holds the "S_i <= sel_i" check and skips an empty partition by +// falling through to the next guard. The check is replaced by an unconditional +// branch when a partition is proven empty (to the next guard) or the caller +// exempts it via avoidPartitionGuard() (to its preheader). All S_i/sel_i are +// materialized once in guard0; the end clamp keeps the "runs at least once" +// iteration in the right partition; live-outs are rebuilt one SSAUpdater each. +// +// A descending (step -1) loop uses the same structure mirrored: partitions run +// high-to-low and the empty test, clamp, and predicates flip (>=/>). +// +// Usage guidelines: +// - Caller bounds must not wrap the induction type. The clamp absorbs a bound +// past the runtime trip count, but a Start +/- offset that overshoots the +// type extreme wraps in the bound arithmetic and cannot be repaired here. +// - Bounds must be loop-invariant: they are expanded in guard0 (the +// preheader), +// so a bound depending on a value defined inside the loop cannot be placed. +// - The partitions must tile the original iteration space exactly -- same +// iterations, same order -- so the split preserves program behaviour. +// - A caller that drops a guard via avoidPartitionGuard() must itself ensure +// that partition runs at least once, or the result is a spurious iteration. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/LoopSplitUtils.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/Analysis/ScalarEvolutionPatternMatch.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/CFG.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/Support/Debug.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Cloning.h" +#include "llvm/Transforms/Utils/LoopUtils.h" +#include "llvm/Transforms/Utils/SSAUpdater.h" +#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" +#include "llvm/Transforms/Utils/ValueMapper.h" +#include + +using namespace llvm; +using namespace llvm::SCEVPatternMatch; + +#define DEBUG_TYPE "loop-split-utils" + +//===----------------------------------------------------------------------===// +// LoopSplitUtils - construction, partition list, induction analysis +//===----------------------------------------------------------------------===// + +/// Per-split() scratch shared by the phase helpers; lives for one split() call. +struct LoopSplitUtils::SplitState { + // Partition 0 reuses the original loop's preheader, exit, and entry guard; + // those blocks live in Partitions[0] rather than being duplicated here. + BasicBlock *FinalExit = nullptr; // where live-outs merge. + Loop *OuterLoop = nullptr; // parent of the new blocks, if any. + PHINode *Induction = nullptr; // the loop's induction variable. + bool Descending = false; // step is negative (loop counts down). + bool LatchComparesPHI = false; // latch compares the PHI, not the step. + + /// A value that must be reconstructed after cloning because it is + /// loop-carried (feeds a later partition), live-out (used after the loop), or + /// both. + struct EscapingValue { + EscapingValue() = default; + EscapingValue(Value *Def) : Def(Def) {} + + /// The value as it exists in partition 0 (the original). + Value *Def = nullptr; + /// The carried header PHI in partition 0, or null if \c Def needs no + /// per-partition start value seeded. + PHINode *CarriedHeaderPHI = nullptr; + /// True if \c Def is used outside the loop and must be merged at the final + /// exit. + bool EscapesOutside = false; + /// \c Def and \c CarriedHeaderPHI cloned into each partition (index 0 is + /// the original; \c PerPartitionPHI[0] is unused). + SmallVector PerPartitionDef; + SmallVector PerPartitionPHI; + }; + + /// Values that must survive across partitions (carried and/or live-out). + SmallVector Escaping; + + EscapingValue &addEscaping(Value *Def) { return Escaping.emplace_back(Def); } +}; + +// Record a new partition with the given inclusive iteration range. +void LoopSplitUtils::addPartition(const SCEV *Start, const SCEV *End) { + Partitions.emplace_back(Start, End); +} + +// Mark a partition so split() emits no entry guard for it. +void LoopSplitUtils::avoidPartitionGuard(unsigned PartitionIndex) { + assert(PartitionIndex < Partitions.size() && + "avoidPartitionGuard() called for an unknown partition"); + Partitions[PartitionIndex].Guarded = false; +} + +// Return a partition's original-to-clone map, or null if it has none. +const ValueToValueMapTy * +LoopSplitUtils::getPartitionValueMap(unsigned PartitionIndex) const { + if (PartitionIndex >= Partitions.size()) + return nullptr; + return Partitions[PartitionIndex].VMap.get(); +} + +// Look up the counterpart of an original value in a given partition. +Value *LoopSplitUtils::getPartitionValue(Value *V, + unsigned PartitionIndex) const { + assert(PartitionIndex < getNumPartitions() && "partition index out of range"); + // Partition 0 reuses the original loop: every value maps to itself. + if (PartitionIndex == 0) + return V; + const ValueToValueMapTy *VMap = getPartitionValueMap(PartitionIndex); + if (!VMap) + return nullptr; + return VMap->lookup(V); +} + +// Find the induction variable and the latch operand it is compared against; +// returns the induction's add-recurrence, or null if the loop is unsuitable. +// On success \p LatchIndOperand is set to the compared induction operand. +static const SCEVAddRecExpr *analyzeInduction(Loop *L, ScalarEvolution *SE, + Value *&LatchIndOperand) { + ICmpInst *LatchCmp = L->getLatchCmpInst(); + + // SCEV's induction variable, restricted to a unit-step affine recurrence. + PHINode *Induction = L->getInductionVariable(*SE); + if (!Induction) + return nullptr; + const SCEV *IndSCEV = SE->getSCEV(Induction); + // Match an affine add-recurrence and capture its constant step; accept a unit + // step in either direction: +1 (ascending) or -1 (descending). + const APInt *Step; + if (!match(IndSCEV, m_scev_AffineAddRec(m_SCEV(), m_scev_APInt(Step)))) + return nullptr; + if (!Step->isOne() && !Step->isAllOnes()) + return nullptr; + const auto *AR = cast(IndSCEV); + + // The induction's "next" value (i + 1), produced in the latch. + auto *StepInst = dyn_cast( + Induction->getIncomingValueForBlock(L->getLoopLatch())); + if (!StepInst) + return nullptr; + + // Select the compare operand that is the induction (PHI or its step). + if (LatchCmp->getOperand(0) == Induction || + LatchCmp->getOperand(0) == StepInst) + LatchIndOperand = LatchCmp->getOperand(0); + else if (LatchCmp->getOperand(1) == Induction || + LatchCmp->getOperand(1) == StepInst) + LatchIndOperand = LatchCmp->getOperand(1); + else + return nullptr; + return AR; +} + +// Decide whether the iteration ordering is signed or unsigned; returns the +// signedness, or nullopt if it cannot be proven. +static std::optional computeSignedness(Loop *L, + const SCEVAddRecExpr *IndAR) { + ICmpInst::Predicate P = L->getLatchCmpInst()->getPredicate(); + // A relational predicate gives the ordering directly; for eq/ne fall back to + // the recurrence's no-wrap flags. + if (ICmpInst::isRelational(P)) + return ICmpInst::isSigned(P); + if (IndAR->hasNoSignedWrap()) + return true; + if (IndAR->hasNoUnsignedWrap()) + return false; + LLVM_DEBUG(dbgs() << DEBUG_TYPE + ": cannot prove iteration ordering signedness\n"); + return std::nullopt; +} + +// Check every structural precondition and record the induction analysis. +bool LoopSplitUtils::isLegal() { + // Require a bottom-tested single-exit loop in LCSSA form with a preheader. + if (!L->getLoopPreheader() || !L->getLoopLatch() || !L->getExitingBlock() || + !L->getExitBlock() || L->getExitingBlock() != L->getLoopLatch() || + !L->isLCSSAForm(*DT)) { + LLVM_DEBUG(dbgs() << DEBUG_TYPE ": loop not in expected form\n"); + return false; + } + + // The latch compare must exist and reside in the latch. + ICmpInst *LatchCmp = L->getLatchCmpInst(); + if (!LatchCmp || LatchCmp->getParent() != L->getLoopLatch()) { + LLVM_DEBUG(dbgs() << DEBUG_TYPE ": latch compare not in the loop latch\n"); + return false; + } + + // A computable backedge-taken count fixes the iteration space we rebuild. + const SCEV *BTC = SE->getBackedgeTakenCount(L); + if (isa(BTC)) { + LLVM_DEBUG(dbgs() << DEBUG_TYPE ": loop trip count uncomputable\n"); + return false; + } + + const SCEVAddRecExpr *IndAR = analyzeInduction(L, SE, LatchIndOperand); + if (!IndAR) { + LLVM_DEBUG(dbgs() << DEBUG_TYPE + ": no unique unit-step integer induction\n"); + return false; + } + + std::optional Signed = computeSignedness(L, IndAR); + if (!Signed) + return false; + InductionIsSigned = *Signed; + + InductionEnd = IndAR->evaluateAtIteration(BTC, *SE); + // Start and end must share the induction type; reject any width mismatch. + if (InductionEnd->getType() != IndAR->getStart()->getType()) { + LLVM_DEBUG(dbgs() << DEBUG_TYPE ": induction end/start type mismatch\n"); + return false; + } + return true; +} + +//===----------------------------------------------------------------------===// +// Transform +//===----------------------------------------------------------------------===// + +// Latch "keep iterating" predicate (ascending />=); inclusive +// when the latch compares the step value, strict when it compares the PHI. +static ICmpInst::Predicate continuePredicate(bool Signed, bool Descending, + bool Inclusive) { + if (Descending) + return Inclusive ? (Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE) + : (Signed ? ICmpInst::ICMP_SGT : ICmpInst::ICMP_UGT); + return Inclusive ? (Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE) + : (Signed ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT); +} + +// Guard "enter this partition" predicate: Start <= sel ascending, Start >= sel +// descending. +static ICmpInst::Predicate guardPredicate(bool Signed, bool Descending) { + if (Descending) + return Signed ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; + return Signed ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; +} + +static void buildEntryGuard(BasicBlock *&Preheader, BasicBlock *&EntryGuard, + DominatorTree *DT, LoopInfo *LI); + +// Drive the whole transform: set up scratch state and run each phase in order. +bool LoopSplitUtils::split() { + PHINode *Induction = L->getInductionVariable(*SE); + assert(Induction && "split() requires a successful isLegal()"); + if (getNumPartitions() < 2) + return false; + + if (!L->hasDedicatedExits() && + !formDedicatedExitBlocks(L, DT, LI, /*MSSAU=*/nullptr, + /*PreserveLCSSA=*/true)) + return false; + + SplitState S; + // Partition 0 reuses the original loop; record its preheader/exit/guard up + // front. + PartitionInfo &P0 = Partitions[0]; + P0.Preheader = L->getLoopPreheader(); + P0.Exit = L->getExitBlock(); + P0.SubLoop = L; + P0.LatchIndOp = LatchIndOperand; + S.OuterLoop = LI->getLoopFor(P0.Exit); + S.Induction = Induction; + // Derive the iteration direction and latch shape once, before transforming. + const auto *IndAR = cast(SE->getSCEV(Induction)); + S.Descending = cast(IndAR->getStepRecurrence(*SE)) + ->getValue() + ->isMinusOne(); + S.LatchComparesPHI = (LatchIndOperand == Induction); + + collectEscapingValues(S); + buildEntryGuard(P0.Preheader, P0.GuardBlock, DT, LI); + + // Keep the expander (and its cleaner) alive for the whole transform: the + // bounds it materializes are consumed by the later phases. If we bail before + // committing, the cleaner reclaims the expanded instructions; on success we + // mark them used so they are kept. + SCEVExpander Expander(*SE, DEBUG_TYPE); + SCEVExpanderCleaner ExpanderCleaner(Expander); + expandPartitionBounds(S, Expander); + clonePartitions(S); + chainPartitions(S); + reconstructSSA(S); + ExpanderCleaner.markResultUsed(); + return true; +} + +// Find loop-carried and live-out values and split the final-exit block off the +// loop exit, seeding partition 0's slots for each escaping value. +void LoopSplitUtils::collectEscapingValues(SplitState &S) { + BasicBlock *Latch = L->getLoopLatch(); + BasicBlock *OrigExit = Partitions[0].Exit; + BasicBlock *OrigPreheader = Partitions[0].Preheader; + + // Separate FinalExit from the loop exit. Split at begin() so the LCSSA PHIs + // move into FinalExit (SplitBlock would advance past them). + S.FinalExit = OrigExit->splitBasicBlock(OrigExit->begin(), "ls.final.exit"); + if (S.OuterLoop) + S.OuterLoop->addBasicBlockToLoop(S.FinalExit, *LI); + // splitBasicBlock does not update the dominator tree; the new exit's sole + // predecessor is the original exit block. + DT->addNewBlock(S.FinalExit, OrigExit); + + // (1) Carried values: each non-induction header PHI whose backedge value + // differs from its initial value must resume in later partitions. + DenseMap CarriedDefToEscapingIdx; + for (PHINode &HeaderPHI : L->getHeader()->phis()) { + if (&HeaderPHI == S.Induction) + continue; + Value *CarriedValue = HeaderPHI.getIncomingValueForBlock(Latch); + Value *InitialValue = HeaderPHI.getIncomingValueForBlock(OrigPreheader); + if (CarriedValue == InitialValue) + continue; // invariant and equal to the initial value: nothing to carry. + auto &EV = S.addEscaping(CarriedValue); + EV.CarriedHeaderPHI = &HeaderPHI; + // Track in-loop carried defs so a matching live-out in (2) merges onto + // them. + if (auto *CarriedInst = dyn_cast(CarriedValue); + CarriedInst && L->contains(CarriedInst)) + CarriedDefToEscapingIdx[CarriedValue] = S.Escaping.size() - 1; + } + + // (2) Live-outs: dissolve each LCSSA PHI into its def and mark it escaping, + // merging onto a pass-(1) entry if also carried. Uses are repaired later. + for (PHINode &LCSSAPhi : make_early_inc_range(S.FinalExit->phis())) { + assert(LCSSAPhi.getNumIncomingValues() == 1 && + "exit block not in LCSSA form"); + Value *LiveOutDef = LCSSAPhi.getIncomingValue(0); + auto Existing = CarriedDefToEscapingIdx.find(LiveOutDef); + auto &EV = Existing != CarriedDefToEscapingIdx.end() + ? S.Escaping[Existing->second] + : S.addEscaping(LiveOutDef); + EV.EscapesOutside = true; + LCSSAPhi.replaceAllUsesWith(LiveOutDef); + LCSSAPhi.eraseFromParent(); + } + + // Seed partition 0 with the originals; later partitions are filled when + // cloned. + const unsigned N = getNumPartitions(); + for (auto &EV : S.Escaping) { + EV.PerPartitionDef.assign(N, nullptr); + EV.PerPartitionPHI.assign(N, nullptr); + EV.PerPartitionDef[0] = EV.Def; + EV.PerPartitionPHI[0] = EV.CarriedHeaderPHI; + } +} + +// Insert the entry guard ahead of partition 0's preheader and update the +// dominator tree. On return \p Preheader is the clean preheader and +// \p EntryGuard is the new guard block dominating the chain. +static void buildEntryGuard(BasicBlock *&Preheader, BasicBlock *&EntryGuard, + DominatorTree *DT, LoopInfo *LI) { + // Split the preheader: the upper half becomes the guard dominating the chain, + // the lower half a clean preheader. + BasicBlock *NewPreheader = + SplitBlock(Preheader, Preheader->getTerminator(), DT, LI); + EntryGuard = Preheader; + Preheader = NewPreheader; + // Move the original preheader's name onto the new preheader, then name the + // guard. + Preheader->takeName(EntryGuard); + EntryGuard->setName("ls.guard0"); +} + +// Materialize each partition's start and clamped end in the entry guard and +// flag the partitions that are provably empty at compile time. +void LoopSplitUtils::expandPartitionBounds(SplitState &S, + SCEVExpander &Expander) { + Type *IndTy = S.Induction->getType(); + Instruction *EntryGuardTerm = Partitions[0].GuardBlock->getTerminator(); + + // Expand all partition bounds in the entry guard, which dominates the whole + // chain (a skipped partition bypasses the original preheader). + const unsigned N = getNumPartitions(); + for (unsigned I = 0; I < N; ++I) { + PartitionInfo &P = Partitions[I]; + + // Provably empty when Start overshoots End by exactly one step. + // Compile-time only: a runtime overshoot wraps at the type extreme and + // would falsely enter. + const SCEV *PartWidth = SE->getMinusSCEV(P.StartExpr, P.EndExpr); + if (auto *PartWidthConst = dyn_cast(PartWidth)) { + const APInt &W = PartWidthConst->getAPInt(); + P.Empty = S.Descending ? W.isAllOnes() : W.isOne(); + } + + P.StartVal = Expander.expandCodeFor(P.StartExpr, IndTy, EntryGuardTerm); + + // Clamp the end to the induction end (min ascending, max descending) so a + // short trip count keeps the last iteration in the right partition. + const SCEV *ClampedEndSCEV; + if (S.Descending) + ClampedEndSCEV = InductionIsSigned + ? SE->getSMaxExpr(P.EndExpr, InductionEnd) + : SE->getUMaxExpr(P.EndExpr, InductionEnd); + else + ClampedEndSCEV = InductionIsSigned + ? SE->getSMinExpr(P.EndExpr, InductionEnd) + : SE->getUMinExpr(P.EndExpr, InductionEnd); + P.SelEnd = Expander.expandCodeFor(ClampedEndSCEV, IndTy, EntryGuardTerm); + } +} + +// Pass 1: clone each later partition's sub-loop and create its guard and exit +// blocks (partition 0 reuses the original loop). +void LoopSplitUtils::clonePartitions(SplitState &S) { + Function &F = *L->getHeader()->getParent(); + LLVMContext &Ctx = F.getContext(); + + const unsigned N = getNumPartitions(); + // Partition 0 reuses the original loop; clone the rest off its preheader. + BasicBlock *OrigPreheader = Partitions[0].Preheader; + + for (unsigned I = 1; I < N; ++I) { + PartitionInfo &P = Partitions[I]; + // Persist this partition's original-to-clone map so callers can later + // query the counterpart of an original loop value (getPartitionValue()). + P.VMap = std::make_unique(); + ValueToValueMapTy &VMap = *P.VMap; + SmallVector ClonedBlocks; + Loop *PL = cloneLoopWithPreheader(S.FinalExit, OrigPreheader, L, VMap, + ".ls" + Twine(I), LI, DT, ClonedBlocks); + remapInstructionsInBlocks(ClonedBlocks, VMap); + BasicBlock *PHi = PL->getLoopPreheader(); + + BasicBlock *Exiti = + BasicBlock::Create(Ctx, "ls.exit" + Twine(I), &F, S.FinalExit); + BasicBlock *Guardi = + BasicBlock::Create(Ctx, "ls.guard" + Twine(I), &F, PHi); + if (S.OuterLoop) { + S.OuterLoop->addBasicBlockToLoop(Exiti, *LI); + S.OuterLoop->addBasicBlockToLoop(Guardi, *LI); + } + // Placeholder terminators; both are re-pointed at the merge in pass 2. + UncondBrInst::Create(S.FinalExit, Exiti); + UncondBrInst::Create(S.FinalExit, Guardi); + + // Seed the clone's induction PHI with this partition's start value. + auto *ClonedInduction = cast(VMap[S.Induction]); + ClonedInduction->setIncomingValueForBlock(PHi, P.StartVal); + + P.GuardBlock = Guardi; + P.Preheader = PHi; + P.Exit = Exiti; + P.SubLoop = PL; + P.LatchIndOp = VMap.lookup_or(LatchIndOperand, LatchIndOperand); + + for (auto &EV : S.Escaping) { + EV.PerPartitionDef[I] = VMap.lookup_or(EV.Def, EV.Def); + if (EV.CarriedHeaderPHI) + EV.PerPartitionPHI[I] = cast(VMap[EV.CarriedHeaderPHI]); + } + } +} + +// Replace a partition's latch test so it iterates only within [start, SelEnd]. +static void rewriteLatch(Loop *PL, Value *IndOp, Value *SelEnd, + BasicBlock *Exit, bool Signed, bool Descending, + bool LatchComparesPHI) { + auto *Term = cast(PL->getLoopLatch()->getTerminator()); + auto *Cmp = cast(Term->getCondition()); + IRBuilder<> B(Cmp); + Value *Bound = SelEnd; + if (Bound->getType() != IndOp->getType()) + Bound = B.CreateIntCast(Bound, IndOp->getType(), Signed); + // Strict when the PHI itself is compared, inclusive when the step value is. + ICmpInst::Predicate Pred = continuePredicate(Signed, Descending, + /*Inclusive=*/!LatchComparesPHI); + Value *NewCmp = B.CreateICmp(Pred, IndOp, Bound, "itr.chk"); + B.SetInsertPoint(Term); + B.CreateCondBr(NewCmp, PL->getHeader(), Exit); + Term->eraseFromParent(); + if (Cmp->use_empty()) + Cmp->eraseFromParent(); +} + +// Pass 2: emit each partition's guard branch, clamp its latch, wire the +// partitions into a chain, and update the dominator tree. +void LoopSplitUtils::chainPartitions(SplitState &S) { + const ICmpInst::Predicate GuardPred = + guardPredicate(InductionIsSigned, S.Descending); + + // Emit each guard, clamp each latch, and chain partitions; a skipped + // partition falls through to the next guard. + const unsigned N = getNumPartitions(); + + // Enters unconditionally when the caller opted out of the guard and the + // partition is not provably empty; a proven-empty partition always skips. + auto EntersUnconditionally = [](const PartitionInfo &P) { + return !P.Empty && !P.Guarded; + }; + + // Where control goes when partition Idx is skipped or after it finishes: the + // next partition's guard, or the final merge block for the last partition. + auto MergeTargetAfter = [&](unsigned Idx) -> BasicBlock * { + bool IsLastPartition = Idx + 1 == N; + return IsLastPartition ? S.FinalExit : Partitions[Idx + 1].GuardBlock; + }; + + for (unsigned I = 0; I < N; ++I) { + PartitionInfo &P = Partitions[I]; + BasicBlock *MergeAfter = MergeTargetAfter(I); + + Instruction *GuardTerm = P.GuardBlock->getTerminator(); + IRBuilder<> B(GuardTerm); + if (P.Empty) { + // Provably empty: skip to the next partition. The unreachable loop body + // is removed by later passes. + B.CreateBr(MergeAfter); + } else if (!P.Guarded) { + // Caller guaranteed at least one iteration: enter unconditionally. The + // skip edge to MergeAfter is omitted (see DT update below). + B.CreateBr(P.Preheader); + } else { + Value *Enter = B.CreateICmp(GuardPred, P.StartVal, P.SelEnd, "itr.chk"); + B.CreateCondBr(Enter, P.Preheader, MergeAfter); + } + GuardTerm->eraseFromParent(); + + rewriteLatch(P.SubLoop, P.LatchIndOp, P.SelEnd, P.Exit, InductionIsSigned, + S.Descending, S.LatchComparesPHI); + P.Exit->getTerminator()->setSuccessor(0, MergeAfter); + } + + // Patch the dominator tree directly: a merge target is dominated by the prior + // partition's exit when it enters unconditionally, otherwise by its guard. + auto MergeTargetIDom = [&](const PartitionInfo &P) { + return EntersUnconditionally(P) ? P.Exit : P.GuardBlock; + }; + + for (unsigned I = 1; I < N; ++I) { + PartitionInfo &Prev = Partitions[I - 1]; + PartitionInfo &Cur = Partitions[I]; + DT->addNewBlock(Cur.GuardBlock, MergeTargetIDom(Prev)); + DT->changeImmediateDominator(Cur.Preheader, Cur.GuardBlock); + DT->addNewBlock(Cur.Exit, Cur.SubLoop->getLoopLatch()); + } + // The final exit is the last partition's merge target. + DT->changeImmediateDominator(S.FinalExit, MergeTargetIDom(Partitions.back())); +} + +// Rebuild SSA for every escaping value, repairing outside uses and seeding each +// later partition's carried PHI, using one SSAUpdater per value. +void LoopSplitUtils::reconstructSSA(SplitState &S) { + const unsigned N = getNumPartitions(); + for (auto &EV : S.Escaping) { + SSAUpdater Updater; + Updater.Initialize(EV.Def->getType(), EV.Def->getName()); + + // Value before any partition runs: carried PHI's initial value, else + // poison. + Value *Init = EV.CarriedHeaderPHI + ? EV.CarriedHeaderPHI->getIncomingValueForBlock( + Partitions[0].Preheader) + : PoisonValue::get(EV.Def->getType()); + Updater.AddAvailableValue(Partitions[0].GuardBlock, Init); + for (unsigned I = 0; I < N; ++I) + Updater.AddAvailableValue(Partitions[I].Exit, EV.PerPartitionDef[I]); + + // Repair outside uses before the carried-PHI seeds add new in-clone uses. + // make_early_inc_range advances past each use before RewriteUse() unlinks + // it from Def's use-list, so the rewrite cannot invalidate the iteration. + if (EV.EscapesOutside) + for (Use &U : make_early_inc_range(EV.Def->uses())) + if (auto *User = dyn_cast(U.getUser())) + if (!L->contains(User)) + Updater.RewriteUse(U); + + // Seed each later partition's carried PHI from the preceding partitions. + if (EV.CarriedHeaderPHI) + for (unsigned I = 1; I < N; ++I) { + PHINode *CarriedPHI = EV.PerPartitionPHI[I]; + int PreheaderEntryIdx = + CarriedPHI->getBasicBlockIndex(Partitions[I].Preheader); + assert(PreheaderEntryIdx >= 0 && "cloned preheader edge missing"); + Updater.RewriteUse(CarriedPHI->getOperandUse(PreheaderEntryIdx)); + } + } +} diff --git a/llvm/lib/Transforms/Utils/LoopSplitUtilsPass.cpp b/llvm/lib/Transforms/Utils/LoopSplitUtilsPass.cpp new file mode 100644 index 0000000000000..ccdc42bc64ed7 --- /dev/null +++ b/llvm/lib/Transforms/Utils/LoopSplitUtilsPass.cpp @@ -0,0 +1,148 @@ +//===- LoopSplitUtilsPass.cpp - Test driver for LoopSplitUtils ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass drives LoopSplitUtils from `opt` for testing. For every eligible +// loop it builds partitions from the -loop-split-points offsets and splits the +// loop. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Utils/LoopSplitUtilsPass.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" +#include "llvm/Analysis/ScalarEvolutionPatternMatch.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/Transforms/Utils/LoopSplitUtils.h" + +using namespace llvm; +using namespace llvm::SCEVPatternMatch; + +#define DEBUG_TYPE "loop-split-utils" + +static cl::list + SplitPoints("loop-split-points", + cl::desc("Iteration offsets (relative to the induction start) " + "at which to split each loop"), + cl::CommaSeparated); + +static cl::list UnguardedPartitions( + "loop-split-unguarded", + cl::desc("Partition indices whose entry guard is omitted (the caller " + "guarantees they run at least one iteration)"), + cl::CommaSeparated); + +/// Build the partition list for \p L from the command-line split offsets and +/// run the transform. Returns true if the loop was split. +static bool splitLoop(Loop *L, ScalarEvolution &SE, DominatorTree &DT, + LoopInfo &LI) { + LoopSplitUtils LSU(L, &LI, &SE, &DT); + if (!LSU.isLegal()) { + LLVM_DEBUG(dbgs() << DEBUG_TYPE ": loop is not legal for splitting\n"); + return false; + } + + const SCEV *IndVarSCEV = SE.getSCEV(LSU.getInductionVariable()); + const SCEV *Start; + const APInt *StepC; + if (!match(IndVarSCEV, + m_scev_AffineAddRec(m_SCEV(Start), m_scev_APInt(StepC)))) + return false; + auto *IndAR = cast(IndVarSCEV); + + const SCEV *BTC = SE.getBackedgeTakenCount(L); + const SCEV *End = IndAR->evaluateAtIteration(BTC, SE); + Type *Ty = Start->getType(); + if (End->getType() != Ty) + End = SE.getTruncateExpr(End, Ty); + + // Build boundaries in iteration order, stepping away from Start by each + // offset (down for a descending loop). Each offset opens a new partition at + // iteration `Start +/- offset`; the previous partition ends one step before. + bool Descending = StepC->isAllOnes(); + + const SCEV *PrevStart = Start; + const SCEV *One = SE.getOne(Ty); + for (unsigned Offset : SplitPoints) { + const SCEV *Off = SE.getConstant(Ty, Offset); + const SCEV *Point = + Descending ? SE.getMinusSCEV(Start, Off) : SE.getAddExpr(Start, Off); + const SCEV *PrevEnd = + Descending ? SE.getAddExpr(Point, One) : SE.getMinusSCEV(Point, One); + LSU.addPartition(PrevStart, PrevEnd); + PrevStart = Point; + } + // The final partition runs to the iteration-space end. + LSU.addPartition(PrevStart, End); + + // Suppress guards for the partitions the caller listed (out-of-range indices + // are ignored). + for (unsigned Idx : UnguardedPartitions) + if (Idx < LSU.getNumPartitions()) + LSU.avoidPartitionGuard(Idx); + + if (LSU.getNumPartitions() < 2) + return false; + + // Snapshot the original loop's named instructions before the transform so we + // can query their per-partition counterparts afterwards (handles track any + // that the transform deletes). Only used to print the debug map below. + [[maybe_unused]] SmallVector OrigValues; + LLVM_DEBUG({ + for (BasicBlock *BB : L->blocks()) + for (Instruction &I : *BB) + if (I.hasName()) + OrigValues.push_back(&I); + }); + + if (!LSU.split()) + return false; + + LLVM_DEBUG({ + const unsigned N = LSU.getNumPartitions(); + for (unsigned P = 0; P < N; ++P) { + dbgs() << "LS-MAP partition " << P << ":\n"; + for (WeakTrackingVH &VH : OrigValues) { + if (!VH) + continue; + Value *M = LSU.getPartitionValue(VH, P); + dbgs() << "LS-MAP " << VH->getName() << " -> " + << (M ? M->getName() : "") << "\n"; + } + } + }); + return true; +} + +PreservedAnalyses LoopSplitUtilsPass::run(Function &F, + FunctionAnalysisManager &AM) { + if (SplitPoints.empty()) + return PreservedAnalyses::all(); + + auto &LI = AM.getResult(F); + auto &SE = AM.getResult(F); + auto &DT = AM.getResult(F); + + // Collect the original top-level loops up front; the transform creates new + // sub-loops that we must not revisit. + SmallVector Worklist(LI.begin(), LI.end()); + + bool Changed = false; + for (Loop *L : Worklist) { + SE.forgetLoop(L); + Changed |= splitLoop(L, SE, DT, LI); + } + + return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} diff --git a/llvm/test/Transforms/LoopSplit/basic.ll b/llvm/test/Transforms/LoopSplit/basic.ll new file mode 100644 index 0000000000000..69b17f76f8f0b --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/basic.ll @@ -0,0 +1,61 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=50 -S < %s | FileCheck %s + +; A single counted loop split into two partitions at iteration 50. + +define void @basic(ptr %a, i64 %n) { +; CHECK-LABEL: define void @basic( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*:]] +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 49) +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 0, [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: store i64 [[I]], ptr [[P]], align 4 +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 [[I_NEXT]], [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 50, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 50, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: store i64 [[I_LS1]], ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK3:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK3]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: ret void +; +; Each partition's latch is clamped to min(partitionEnd, loopEnd), emitted as a +; signed-min intrinsic for a signed induction. +; A later partition is entered only if a runtime check passes, and a second, +; cloned loop drives the remaining iterations. +; The transform builds a dedicated final-exit block joining all partitions. +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + store i64 %i, ptr %p + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopSplit/constant-trip-count.ll b/llvm/test/Transforms/LoopSplit/constant-trip-count.ll new file mode 100644 index 0000000000000..19d553767e71a --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/constant-trip-count.ll @@ -0,0 +1,89 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=50 -S < %s \ +; RUN: | FileCheck %s --check-prefixes=VALID +; RUN: opt -passes=loop-split-utils -loop-split-points=200 -S < %s \ +; RUN: | FileCheck %s --check-prefixes=INVALID + +; With a constant trip count (100) the per-partition bound and clamp arithmetic +; folds at compile time. A split point of 50 lands inside [0,99] and yields two +; non-empty partitions [0,49] and [50,99]. A split point of 200 is past the last +; iteration, so the second partition's `Start <= End` guard folds to a constant +; false and that partition is never entered. + +define void @constant_tc(ptr %a) { +; VALID-LABEL: define void @constant_tc( +; VALID-SAME: ptr [[A:%.*]]) { +; VALID-NEXT: [[LS_GUARD0:.*:]] +; VALID-NEXT: br i1 true, label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; VALID: [[ENTRY]]: +; VALID-NEXT: br label %[[LOOP:.*]] +; VALID: [[LOOP]]: +; VALID-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; VALID-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; VALID-NEXT: store i64 [[I]], ptr [[P]], align 4 +; VALID-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; VALID-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 [[I_NEXT]], 49 +; VALID-NEXT: br i1 [[ITR_CHK]], label %[[LOOP]], label %[[EXIT:.*]] +; VALID: [[EXIT]]: +; VALID-NEXT: br label %[[LS_GUARD1]] +; VALID: [[LS_GUARD1]]: +; VALID-NEXT: br i1 true, label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; VALID: [[ENTRY_LS1]]: +; VALID-NEXT: br label %[[LOOP_LS1:.*]] +; VALID: [[LOOP_LS1]]: +; VALID-NEXT: [[I_LS1:%.*]] = phi i64 [ 50, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; VALID-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; VALID-NEXT: store i64 [[I_LS1]], ptr [[P_LS1]], align 4 +; VALID-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; VALID-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 [[I_NEXT_LS1]], 99 +; VALID-NEXT: br i1 [[ITR_CHK1]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; VALID: [[LS_EXIT1]]: +; VALID-NEXT: br label %[[LS_FINAL_EXIT]] +; VALID: [[LS_FINAL_EXIT]]: +; VALID-NEXT: ret void +; +; INVALID-LABEL: define void @constant_tc( +; INVALID-SAME: ptr [[A:%.*]]) { +; INVALID-NEXT: [[LS_GUARD0:.*:]] +; INVALID-NEXT: br i1 true, label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; INVALID: [[ENTRY]]: +; INVALID-NEXT: br label %[[LOOP:.*]] +; INVALID: [[LOOP]]: +; INVALID-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; INVALID-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; INVALID-NEXT: store i64 [[I]], ptr [[P]], align 4 +; INVALID-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; INVALID-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 [[I_NEXT]], 99 +; INVALID-NEXT: br i1 [[ITR_CHK]], label %[[LOOP]], label %[[EXIT:.*]] +; INVALID: [[EXIT]]: +; INVALID-NEXT: br label %[[LS_GUARD1]] +; INVALID: [[LS_GUARD1]]: +; INVALID-NEXT: br i1 false, label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; INVALID: [[ENTRY_LS1]]: +; INVALID-NEXT: br label %[[LOOP_LS1:.*]] +; INVALID: [[LOOP_LS1]]: +; INVALID-NEXT: [[I_LS1:%.*]] = phi i64 [ 200, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; INVALID-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; INVALID-NEXT: store i64 [[I_LS1]], ptr [[P_LS1]], align 4 +; INVALID-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; INVALID-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 [[I_NEXT_LS1]], 99 +; INVALID-NEXT: br i1 [[ITR_CHK1]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; INVALID: [[LS_EXIT1]]: +; INVALID-NEXT: br label %[[LS_FINAL_EXIT]] +; INVALID: [[LS_FINAL_EXIT]]: +; INVALID-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + store i64 %i, ptr %p + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, 100 + br i1 %c, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopSplit/descending.ll b/llvm/test/Transforms/LoopSplit/descending.ll new file mode 100644 index 0000000000000..2acddcd7e93b2 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/descending.ll @@ -0,0 +1,60 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=50 -S < %s | FileCheck %s + +; A signed counting-down loop (step -1) split into two partitions. The guard and +; latch predicates flip to >=, and the end clamp uses smax instead of smin. + +define void @descending(ptr %a, i64 %n) { +; CHECK-LABEL: define void @descending( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*:]] +; CHECK-NEXT: [[TMP0:%.*]] = add i64 [[N]], -49 +; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[N]], -1 +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP1]], i64 -1) +; CHECK-NEXT: [[TMP2:%.*]] = add nsw i64 [[SMIN]], 1 +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[TMP0]], i64 [[TMP2]]) +; CHECK-NEXT: [[TMP3:%.*]] = add i64 [[N]], -50 +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sge i64 [[N]], [[SMAX]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ [[N]], %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: store i64 [[I]], ptr [[P]], align 4 +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], -1 +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sge i64 [[I_NEXT]], [[SMAX]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sge i64 [[TMP3]], [[TMP2]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ [[TMP3]], %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: store i64 [[I_LS1]], ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], -1 +; CHECK-NEXT: [[ITR_CHK3:%.*]] = icmp sge i64 [[I_NEXT_LS1]], [[TMP2]] +; CHECK-NEXT: br i1 [[ITR_CHK3]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i64 [ %n, %entry ], [ %i.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + store i64 %i, ptr %p + %i.next = add i64 %i, -1 + %c = icmp sge i64 %i.next, 0 + br i1 %c, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopSplit/empty-leading-partition.ll b/llvm/test/Transforms/LoopSplit/empty-leading-partition.ll new file mode 100644 index 0000000000000..d547ed7df6c93 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/empty-leading-partition.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=0 -S < %s | FileCheck %s + +; A leading split point of 0 makes partition 0 empty ([Start, Start-1]). The +; loop is bottom-tested, so without an entry guard partition 0 would still run +; one iteration and the loop would execute its first iteration twice. The empty +; partition is detected at compile time (its width S - E folds to a constant) +; and skipped: partition 0's guard unconditionally falls through to partition 1, +; which runs the whole iteration space and seeds from the chain-entry value (0). + +define i64 @reduction(ptr %a, i64 %n) { +; CHECK-LABEL: define i64 @reduction( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*]]: +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: br label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 4 +; CHECK-NEXT: [[SUM_NEXT]] = add i64 [[SUM]], [[V]] +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 [[I_NEXT]], -1 +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[SUM_NEXT4:%.*]] = phi i64 [ [[SUM_NEXT]], %[[EXIT]] ], [ 0, %[[LS_GUARD0]] ] +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 0, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 0, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[SUM_LS1:%.*]] = phi i64 [ [[SUM_NEXT4]], %[[ENTRY_LS1]] ], [ [[SUM_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: [[V_LS1:%.*]] = load i64, ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[SUM_NEXT_LS1]] = add i64 [[SUM_LS1]], [[V_LS1]] +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: [[SUM_NEXT3:%.*]] = phi i64 [ [[SUM_NEXT_LS1]], %[[LS_EXIT1]] ], [ [[SUM_NEXT4]], %[[LS_GUARD1]] ] +; CHECK-NEXT: ret i64 [[SUM_NEXT3]] +; +; The empty leading partition is never entered: an unconditional fall-through. +; The original loop body is now dead (unreachable), proving it is not executed. +; The surviving partition runs the full range, seeded from the chain-entry value. +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %sum = phi i64 [ 0, %entry ], [ %sum.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + %v = load i64, ptr %p + %sum.next = add i64 %sum, %v + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + %sum.lcssa = phi i64 [ %sum.next, %loop ] + ret i64 %sum.lcssa +} diff --git a/llvm/test/Transforms/LoopSplit/four-partitions.ll b/llvm/test/Transforms/LoopSplit/four-partitions.ll new file mode 100644 index 0000000000000..8a0d7009fb973 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/four-partitions.ll @@ -0,0 +1,88 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=100,200,300 -S < %s | FileCheck %s + +; A minimal counted loop split into four partitions: [0,99], [100,199], +; [200,299] and [300,n-1]. Three clones (loop.ls1, loop.ls2, loop.ls3) chain +; through their guards into a single dedicated final-exit block. + +define void @four(ptr %a, i64 %n) { +; CHECK-LABEL: define void @four( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*:]] +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: [[SMIN2:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 99) +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 199) +; CHECK-NEXT: [[SMIN1:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 299) +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 0, [[SMIN2]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: store i64 [[I]], ptr [[P]], align 4 +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 [[I_NEXT]], [[SMIN2]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 100, [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[ENTRY_LS1:.*]], label %[[LS_GUARD2:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 100, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: store i64 [[I_LS1]], ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK3:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK3]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_GUARD2]] +; CHECK: [[LS_GUARD2]]: +; CHECK-NEXT: [[ITR_CHK4:%.*]] = icmp sle i64 200, [[SMIN1]] +; CHECK-NEXT: br i1 [[ITR_CHK4]], label %[[ENTRY_LS2:.*]], label %[[LS_GUARD3:.*]] +; CHECK: [[ENTRY_LS2]]: +; CHECK-NEXT: br label %[[LOOP_LS2:.*]] +; CHECK: [[LOOP_LS2]]: +; CHECK-NEXT: [[I_LS2:%.*]] = phi i64 [ 200, %[[ENTRY_LS2]] ], [ [[I_NEXT_LS2:%.*]], %[[LOOP_LS2]] ] +; CHECK-NEXT: [[P_LS2:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS2]] +; CHECK-NEXT: store i64 [[I_LS2]], ptr [[P_LS2]], align 4 +; CHECK-NEXT: [[I_NEXT_LS2]] = add i64 [[I_LS2]], 1 +; CHECK-NEXT: [[ITR_CHK5:%.*]] = icmp sle i64 [[I_NEXT_LS2]], [[SMIN1]] +; CHECK-NEXT: br i1 [[ITR_CHK5]], label %[[LOOP_LS2]], label %[[LS_EXIT2:.*]] +; CHECK: [[LS_EXIT2]]: +; CHECK-NEXT: br label %[[LS_GUARD3]] +; CHECK: [[LS_GUARD3]]: +; CHECK-NEXT: [[ITR_CHK6:%.*]] = icmp sle i64 300, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK6]], label %[[ENTRY_LS3:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS3]]: +; CHECK-NEXT: br label %[[LOOP_LS3:.*]] +; CHECK: [[LOOP_LS3]]: +; CHECK-NEXT: [[I_LS3:%.*]] = phi i64 [ 300, %[[ENTRY_LS3]] ], [ [[I_NEXT_LS3:%.*]], %[[LOOP_LS3]] ] +; CHECK-NEXT: [[P_LS3:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS3]] +; CHECK-NEXT: store i64 [[I_LS3]], ptr [[P_LS3]], align 4 +; CHECK-NEXT: [[I_NEXT_LS3]] = add i64 [[I_LS3]], 1 +; CHECK-NEXT: [[ITR_CHK7:%.*]] = icmp sle i64 [[I_NEXT_LS3]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK7]], label %[[LOOP_LS3]], label %[[LS_EXIT3:.*]] +; CHECK: [[LS_EXIT3]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + store i64 %i, ptr %p + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopSplit/multiple-partitions.ll b/llvm/test/Transforms/LoopSplit/multiple-partitions.ll new file mode 100644 index 0000000000000..ac4fcbda6dc82 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/multiple-partitions.ll @@ -0,0 +1,75 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=50,100 -S < %s | FileCheck %s + +; A loop split into three partitions: [0,49], [50,99], [100,n-1]. + +define void @three_way(ptr %a, i64 %n) { +; CHECK-LABEL: define void @three_way( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*:]] +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 49) +; CHECK-NEXT: [[SMIN1:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 99) +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 0, [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: store i64 [[I]], ptr [[P]], align 4 +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 [[I_NEXT]], [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[ITR_CHK3:%.*]] = icmp sle i64 50, [[SMIN1]] +; CHECK-NEXT: br i1 [[ITR_CHK3]], label %[[ENTRY_LS1:.*]], label %[[LS_GUARD2:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 50, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: store i64 [[I_LS1]], ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK4:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[SMIN1]] +; CHECK-NEXT: br i1 [[ITR_CHK4]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_GUARD2]] +; CHECK: [[LS_GUARD2]]: +; CHECK-NEXT: [[ITR_CHK5:%.*]] = icmp sle i64 100, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK5]], label %[[ENTRY_LS2:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS2]]: +; CHECK-NEXT: br label %[[LOOP_LS2:.*]] +; CHECK: [[LOOP_LS2]]: +; CHECK-NEXT: [[I_LS2:%.*]] = phi i64 [ 100, %[[ENTRY_LS2]] ], [ [[I_NEXT_LS2:%.*]], %[[LOOP_LS2]] ] +; CHECK-NEXT: [[P_LS2:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS2]] +; CHECK-NEXT: store i64 [[I_LS2]], ptr [[P_LS2]], align 4 +; CHECK-NEXT: [[I_NEXT_LS2]] = add i64 [[I_LS2]], 1 +; CHECK-NEXT: [[ITR_CHK6:%.*]] = icmp sle i64 [[I_NEXT_LS2]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK6]], label %[[LOOP_LS2]], label %[[LS_EXIT2:.*]] +; CHECK: [[LS_EXIT2]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: ret void +; +; Each partition is entered only if a runtime min/iteration check passes; the +; min(partitionEnd, loopEnd) clamp is a signed-min intrinsic. +; The original loop plus two clones cover [0,49], [50,99] and [100,n-1], and a +; dedicated final-exit block joins all three partitions. +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + store i64 %i, ptr %p + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopSplit/nested-loop.ll b/llvm/test/Transforms/LoopSplit/nested-loop.ll new file mode 100644 index 0000000000000..a4cb06feb7aa2 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/nested-loop.ll @@ -0,0 +1,88 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=50 -S < %s | FileCheck %s + +; Only the outer (top-level) loop is a split candidate. Splitting it clones the +; whole loop body, including the inner loop, into the later partition. This +; exercises that cloning a loop containing a subloop does not crash and that the +; inner loop is duplicated wholesale rather than split. + +define void @nested(ptr %a, i64 %n, i64 %m) { +; CHECK-LABEL: define void @nested( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]], i64 [[M:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*:]] +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 49) +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 0, [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[OUTER_HEADER:.*]] +; CHECK: [[OUTER_HEADER]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[OUTER_LATCH:.*]] ] +; CHECK-NEXT: br label %[[INNER_HEADER:.*]] +; CHECK: [[INNER_HEADER]]: +; CHECK-NEXT: [[J:%.*]] = phi i64 [ 0, %[[OUTER_HEADER]] ], [ [[J_NEXT:%.*]], %[[INNER_HEADER]] ] +; CHECK-NEXT: [[BASE:%.*]] = mul i64 [[I]], [[M]] +; CHECK-NEXT: [[IDX:%.*]] = add i64 [[BASE]], [[J]] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[IDX]] +; CHECK-NEXT: store i64 [[IDX]], ptr [[P]], align 4 +; CHECK-NEXT: [[J_NEXT]] = add i64 [[J]], 1 +; CHECK-NEXT: [[IC:%.*]] = icmp slt i64 [[J_NEXT]], [[M]] +; CHECK-NEXT: br i1 [[IC]], label %[[INNER_HEADER]], label %[[OUTER_LATCH]] +; CHECK: [[OUTER_LATCH]]: +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 [[I_NEXT]], [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[OUTER_HEADER]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 50, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[OUTER_HEADER_LS1:.*]] +; CHECK: [[OUTER_HEADER_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 50, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[OUTER_LATCH_LS1:.*]] ] +; CHECK-NEXT: br label %[[INNER_HEADER_LS1:.*]] +; CHECK: [[INNER_HEADER_LS1]]: +; CHECK-NEXT: [[J_LS1:%.*]] = phi i64 [ 0, %[[OUTER_HEADER_LS1]] ], [ [[J_NEXT_LS1:%.*]], %[[INNER_HEADER_LS1]] ] +; CHECK-NEXT: [[BASE_LS1:%.*]] = mul i64 [[I_LS1]], [[M]] +; CHECK-NEXT: [[IDX_LS1:%.*]] = add i64 [[BASE_LS1]], [[J_LS1]] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[IDX_LS1]] +; CHECK-NEXT: store i64 [[IDX_LS1]], ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[J_NEXT_LS1]] = add i64 [[J_LS1]], 1 +; CHECK-NEXT: [[IC_LS1:%.*]] = icmp slt i64 [[J_NEXT_LS1]], [[M]] +; CHECK-NEXT: br i1 [[IC_LS1]], label %[[INNER_HEADER_LS1]], label %[[OUTER_LATCH_LS1]] +; CHECK: [[OUTER_LATCH_LS1]]: +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK3:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK3]], label %[[OUTER_HEADER_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: ret void +; +entry: + br label %outer.header + +outer.header: + %i = phi i64 [ 0, %entry ], [ %i.next, %outer.latch ] + br label %inner.header + +inner.header: + %j = phi i64 [ 0, %outer.header ], [ %j.next, %inner.header ] + %base = mul i64 %i, %m + %idx = add i64 %base, %j + %p = getelementptr i64, ptr %a, i64 %idx + store i64 %idx, ptr %p + %j.next = add i64 %j, 1 + %ic = icmp slt i64 %j.next, %m + br i1 %ic, label %inner.header, label %outer.latch + +outer.latch: + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %outer.header, label %exit + +exit: + ret void +} diff --git a/llvm/test/Transforms/LoopSplit/optional-guard.ll b/llvm/test/Transforms/LoopSplit/optional-guard.ll new file mode 100644 index 0000000000000..82ea84e8dce11 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/optional-guard.ll @@ -0,0 +1,70 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=4 -loop-split-unguarded=0 \ +; RUN: -verify-dom-info -verify-loop-info -S < %s | FileCheck %s + +; Per-partition entry guards are optional. The split loop is bottom-tested, so +; by default every partition is guarded by a `Start <= End` check that skips a +; zero-iteration partition. A caller that can prove a partition runs at least +; once may omit its guard (here partition 0, via -loop-split-unguarded=0): that +; guard then enters the sub-loop unconditionally and has no skip edge to the +; next partition. Partition 1 keeps its guard. The dominator tree is maintained +; incrementally for the missing skip edge, so -verify-dom-info must pass. + +define i64 @reduction(ptr %a, i64 %n) { +; CHECK-LABEL: define i64 @reduction( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*:]] +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 3) +; CHECK-NEXT: br label %[[ENTRY:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 4 +; CHECK-NEXT: [[SUM_NEXT]] = add i64 [[SUM]], [[V]] +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 [[I_NEXT]], [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1:.*]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 4, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 4, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[SUM_LS1:%.*]] = phi i64 [ [[SUM_NEXT]], %[[ENTRY_LS1]] ], [ [[SUM_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: [[V_LS1:%.*]] = load i64, ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[SUM_NEXT_LS1]] = add i64 [[SUM_LS1]], [[V_LS1]] +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: [[SUM_NEXT3:%.*]] = phi i64 [ [[SUM_NEXT_LS1]], %[[LS_EXIT1]] ], [ [[SUM_NEXT]], %[[LS_GUARD1]] ] +; CHECK-NEXT: ret i64 [[SUM_NEXT3]] +; +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %sum = phi i64 [ 0, %entry ], [ %sum.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + %v = load i64, ptr %p + %sum.next = add i64 %sum, %v + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + %sum.lcssa = phi i64 [ %sum.next, %loop ] + ret i64 %sum.lcssa +} diff --git a/llvm/test/Transforms/LoopSplit/partition-value-map.ll b/llvm/test/Transforms/LoopSplit/partition-value-map.ll new file mode 100644 index 0000000000000..a7f5df90b156f --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/partition-value-map.ll @@ -0,0 +1,44 @@ +; REQUIRES: asserts +; RUN: opt -passes=loop-split-utils -loop-split-points=4,8 \ +; RUN: -debug-only=loop-split-utils -disable-output < %s 2>&1 | FileCheck %s + +; LoopSplitUtils preserves the original-to-clone value map for every partition +; and exposes it through getPartitionValue(). Partition 0 reuses the original +; loop (identity mapping); later partitions return the cloned counterpart, named +; with a `.lsN` suffix. The latch compare (%c) is rewritten rather than cloned, +; so it has no surviving counterpart and is omitted from every partition. + +define i32 @sum(ptr %a, i32 %n) { +entry: + br label %loop + +loop: + %i = phi i32 [ 0, %entry ], [ %i.next, %loop ] + %acc = phi i32 [ 0, %entry ], [ %acc.next, %loop ] + %p = getelementptr inbounds i32, ptr %a, i32 %i + %v = load i32, ptr %p + %acc.next = add i32 %acc, %v + %i.next = add i32 %i, 1 + %c = icmp slt i32 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + %lcssa = phi i32 [ %acc.next, %loop ] + ret i32 %lcssa +} + +; CHECK: LS-MAP partition 0: +; CHECK: i -> i +; CHECK: acc -> acc +; CHECK: acc.next -> acc.next +; CHECK: i.next -> i.next +; CHECK: LS-MAP partition 1: +; CHECK: i -> i.ls1 +; CHECK: acc -> acc.ls1 +; CHECK: acc.next -> acc.next.ls1 +; CHECK: i.next -> i.next.ls1 +; CHECK: LS-MAP partition 2: +; CHECK: i -> i.ls2 +; CHECK: acc -> acc.ls2 +; CHECK: acc.next -> acc.next.ls2 +; CHECK: i.next -> i.next.ls2 diff --git a/llvm/test/Transforms/LoopSplit/reduction.ll b/llvm/test/Transforms/LoopSplit/reduction.ll new file mode 100644 index 0000000000000..c9cd745469323 --- /dev/null +++ b/llvm/test/Transforms/LoopSplit/reduction.ll @@ -0,0 +1,71 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -passes=loop-split-utils -loop-split-points=50 -S < %s | FileCheck %s + +; A reduction (live-out) is correctly threaded across the two partitions: the +; second partition seeds its accumulator with the first partition's join value +; and a dedicated final-exit block merges the per-partition results. + +define i64 @reduction(ptr %a, i64 %n) { +; CHECK-LABEL: define i64 @reduction( +; CHECK-SAME: ptr [[A:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: [[LS_GUARD0:.*]]: +; CHECK-NEXT: [[SMAX:%.*]] = call i64 @llvm.smax.i64(i64 [[N]], i64 1) +; CHECK-NEXT: [[TMP0:%.*]] = add nsw i64 [[SMAX]], -1 +; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[TMP0]], i64 49) +; CHECK-NEXT: [[ITR_CHK:%.*]] = icmp sle i64 0, [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK]], label %[[ENTRY:.*]], label %[[LS_GUARD1:.*]] +; CHECK: [[ENTRY]]: +; CHECK-NEXT: br label %[[LOOP:.*]] +; CHECK: [[LOOP]]: +; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[SUM:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[SUM_NEXT:%.*]], %[[LOOP]] ] +; CHECK-NEXT: [[P:%.*]] = getelementptr i64, ptr [[A]], i64 [[I]] +; CHECK-NEXT: [[V:%.*]] = load i64, ptr [[P]], align 4 +; CHECK-NEXT: [[SUM_NEXT]] = add i64 [[SUM]], [[V]] +; CHECK-NEXT: [[I_NEXT]] = add i64 [[I]], 1 +; CHECK-NEXT: [[ITR_CHK1:%.*]] = icmp sle i64 [[I_NEXT]], [[SMIN]] +; CHECK-NEXT: br i1 [[ITR_CHK1]], label %[[LOOP]], label %[[EXIT:.*]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: br label %[[LS_GUARD1]] +; CHECK: [[LS_GUARD1]]: +; CHECK-NEXT: [[SUM_NEXT5:%.*]] = phi i64 [ [[SUM_NEXT]], %[[EXIT]] ], [ 0, %[[LS_GUARD0]] ] +; CHECK-NEXT: [[ITR_CHK2:%.*]] = icmp sle i64 50, [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK2]], label %[[ENTRY_LS1:.*]], label %[[LS_FINAL_EXIT:.*]] +; CHECK: [[ENTRY_LS1]]: +; CHECK-NEXT: br label %[[LOOP_LS1:.*]] +; CHECK: [[LOOP_LS1]]: +; CHECK-NEXT: [[I_LS1:%.*]] = phi i64 [ 50, %[[ENTRY_LS1]] ], [ [[I_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[SUM_LS1:%.*]] = phi i64 [ [[SUM_NEXT5]], %[[ENTRY_LS1]] ], [ [[SUM_NEXT_LS1:%.*]], %[[LOOP_LS1]] ] +; CHECK-NEXT: [[P_LS1:%.*]] = getelementptr i64, ptr [[A]], i64 [[I_LS1]] +; CHECK-NEXT: [[V_LS1:%.*]] = load i64, ptr [[P_LS1]], align 4 +; CHECK-NEXT: [[SUM_NEXT_LS1]] = add i64 [[SUM_LS1]], [[V_LS1]] +; CHECK-NEXT: [[I_NEXT_LS1]] = add i64 [[I_LS1]], 1 +; CHECK-NEXT: [[ITR_CHK3:%.*]] = icmp sle i64 [[I_NEXT_LS1]], [[TMP0]] +; CHECK-NEXT: br i1 [[ITR_CHK3]], label %[[LOOP_LS1]], label %[[LS_EXIT1:.*]] +; CHECK: [[LS_EXIT1]]: +; CHECK-NEXT: br label %[[LS_FINAL_EXIT]] +; CHECK: [[LS_FINAL_EXIT]]: +; CHECK-NEXT: [[SUM_NEXT4:%.*]] = phi i64 [ [[SUM_NEXT_LS1]], %[[LS_EXIT1]] ], [ [[SUM_NEXT5]], %[[LS_GUARD1]] ] +; CHECK-NEXT: ret i64 [[SUM_NEXT4]] +; +; The second partition's entry guard joins the first partition's result with the +; chain-entry value (used when the first partition runs zero iterations). +; The cloned partition seeds its accumulator from that join value. +; A dedicated final-exit block merges the per-partition results. +entry: + br label %loop + +loop: + %i = phi i64 [ 0, %entry ], [ %i.next, %loop ] + %sum = phi i64 [ 0, %entry ], [ %sum.next, %loop ] + %p = getelementptr i64, ptr %a, i64 %i + %v = load i64, ptr %p + %sum.next = add i64 %sum, %v + %i.next = add i64 %i, 1 + %c = icmp slt i64 %i.next, %n + br i1 %c, label %loop, label %exit + +exit: + %sum.lcssa = phi i64 [ %sum.next, %loop ] + ret i64 %sum.lcssa +}