Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 34 additions & 26 deletions llvm/lib/Transforms/Scalar/LoopFlatten.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -835,9 +835,39 @@ static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
return true;
}

static bool VersionLoop(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, const LoopAccessInfo &LAI) {

// Version the loop. The overflow check isn't a runtime pointer check, so we
// pass an empty list of runtime pointer checks, causing LoopVersioning to
// emit 'false' as the branch condition, and add our own check afterwards.
BasicBlock *CheckBlock = FI.OuterLoop->getLoopPreheader();
ArrayRef<RuntimePointerCheck> Checks(nullptr, nullptr);
LoopVersioning LVer(LAI, Checks, FI.OuterLoop, LI, DT, SE);
LVer.versionLoop();

// Check for overflow by calculating the new tripcount using
// umul_with_overflow and then checking if it overflowed.
BranchInst *Br = dyn_cast<BranchInst>(CheckBlock->getTerminator());
if (!Br || !Br->isConditional())
return false;
if (!match(Br->getCondition(), m_Zero()))
return false;
IRBuilder<> Builder(Br);
Value *Call = Builder.CreateIntrinsic(Intrinsic::umul_with_overflow,
FI.OuterTripCount->getType(),
{FI.OuterTripCount, FI.InnerTripCount},
/*FMFSource=*/nullptr, "flatten.mul");
FI.NewTripCount = Builder.CreateExtractValue(Call, 0, "flatten.tripcount");
Value *Overflow = Builder.CreateExtractValue(Call, 1, "flatten.overflow");
Br->setCondition(Overflow);
Comment on lines +857 to +863
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Independent of the change here, but it seems like the loop versioning interface could do with some refactoring to clean things up.

LAI in LoopFlatten is only used to pass to LoopVersioning, but isn't used; LoopVersioning should provide an interface that doesn't require passing it.

Not sure if that would be possible, but hperhaps the wrapping predicates could re-use the logic in PredicatedScalarEvolution as well?

return true;
}

static bool CanWidenIV(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
ScalarEvolution *SE, AssumptionCache *AC,
const TargetTransformInfo *TTI) {
const TargetTransformInfo *TTI,
const LoopAccessInfo &LAI) {
if (!WidenIV) {
LLVM_DEBUG(dbgs() << "Widening the IVs is disabled\n");
return false;
Expand Down Expand Up @@ -916,7 +946,7 @@ static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
return false;

// Check if we can widen the induction variables to avoid overflow checks.
bool CanFlatten = CanWidenIV(FI, DT, LI, SE, AC, TTI);
bool CanFlatten = CanWidenIV(FI, DT, LI, SE, AC, TTI, LAI);

// It can happen that after widening of the IV, flattening may not be
// possible/happening, e.g. when it is deemed unprofitable. So bail here if
Expand Down Expand Up @@ -961,30 +991,8 @@ static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
return false;
}
LLVM_DEBUG(dbgs() << "Multiply might overflow, versioning loop\n");

// Version the loop. The overflow check isn't a runtime pointer check, so we
// pass an empty list of runtime pointer checks, causing LoopVersioning to
// emit 'false' as the branch condition, and add our own check afterwards.
BasicBlock *CheckBlock = FI.OuterLoop->getLoopPreheader();
ArrayRef<RuntimePointerCheck> Checks(nullptr, nullptr);
LoopVersioning LVer(LAI, Checks, FI.OuterLoop, LI, DT, SE);
LVer.versionLoop();

// Check for overflow by calculating the new tripcount using
// umul_with_overflow and then checking if it overflowed.
BranchInst *Br = cast<BranchInst>(CheckBlock->getTerminator());
assert(Br->isConditional() &&
"Expected LoopVersioning to generate a conditional branch");
assert(match(Br->getCondition(), m_Zero()) &&
"Expected branch condition to be false");
IRBuilder<> Builder(Br);
Value *Call = Builder.CreateIntrinsic(
Intrinsic::umul_with_overflow, FI.OuterTripCount->getType(),
{FI.OuterTripCount, FI.InnerTripCount},
/*FMFSource=*/nullptr, "flatten.mul");
FI.NewTripCount = Builder.CreateExtractValue(Call, 0, "flatten.tripcount");
Value *Overflow = Builder.CreateExtractValue(Call, 1, "flatten.overflow");
Br->setCondition(Overflow);
bool LoopIsVersioned = VersionLoop(FI, DT, LI, SE, LAI);
assert(LoopIsVersioned && "Failed to version loop");
} else {
LLVM_DEBUG(dbgs() << "Multiply cannot overflow, modifying loop in-place\n");
}
Expand Down
Loading
Loading