Expand Up
@@ -193,7 +193,9 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, int OptLevel,
Optional<unsigned > UserThreshold, Optional<unsigned > UserCount,
Optional<bool > UserAllowPartial, Optional<bool > UserRuntime,
Optional<bool > UserUpperBound, Optional<unsigned > UserFullUnrollMaxCount) {
Optional<bool > UserUpperBound, Optional<bool > UserAllowPeeling,
Optional<bool > UserAllowProfileBasedPeeling,
Optional<unsigned > UserFullUnrollMaxCount) {
TargetTransformInfo::UnrollingPreferences UP;
// Set up the defaults
Expand All
@@ -204,6 +206,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.PartialThreshold = 150 ;
UP.PartialOptSizeThreshold = 0 ;
UP.Count = 0 ;
UP.PeelCount = 0 ;
UP.DefaultUnrollRuntimeCount = 8 ;
UP.MaxCount = std::numeric_limits<unsigned >::max ();
UP.FullUnrollMaxCount = std::numeric_limits<unsigned >::max ();
Expand All
@@ -215,7 +218,10 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.AllowExpensiveTripCount = false ;
UP.Force = false ;
UP.UpperBound = false ;
UP.AllowPeeling = true ;
UP.AllowLoopNestsPeeling = false ;
UP.UnrollAndJam = false ;
UP.PeelProfiledIterations = true ;
UP.UnrollAndJamInnerLoopThreshold = 60 ;
UP.MaxIterationsCountToAnalyze = UnrollMaxIterationsCountToAnalyze;
Expand Down
Expand Up
@@ -243,6 +249,8 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.MaxCount = UnrollMaxCount;
if (UnrollFullMaxCount.getNumOccurrences () > 0 )
UP.FullUnrollMaxCount = UnrollFullMaxCount;
if (UnrollPeelCount.getNumOccurrences () > 0 )
UP.PeelCount = UnrollPeelCount;
if (UnrollAllowPartial.getNumOccurrences () > 0 )
UP.Partial = UnrollAllowPartial;
if (UnrollAllowRemainder.getNumOccurrences () > 0 )
Expand All
@@ -251,6 +259,10 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.Runtime = UnrollRuntime;
if (UnrollMaxUpperBound == 0 )
UP.UpperBound = false ;
if (UnrollAllowPeeling.getNumOccurrences () > 0 )
UP.AllowPeeling = UnrollAllowPeeling;
if (UnrollAllowLoopNestsPeeling.getNumOccurrences () > 0 )
UP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
if (UnrollUnrollRemainder.getNumOccurrences () > 0 )
UP.UnrollRemainder = UnrollUnrollRemainder;
if (UnrollMaxIterationsCountToAnalyze.getNumOccurrences () > 0 )
Expand All
@@ -269,39 +281,16 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
UP.Runtime = *UserRuntime;
if (UserUpperBound.hasValue ())
UP.UpperBound = *UserUpperBound;
if (UserAllowPeeling.hasValue ())
UP.AllowPeeling = *UserAllowPeeling;
if (UserAllowProfileBasedPeeling.hasValue ())
UP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
if (UserFullUnrollMaxCount.hasValue ())
UP.FullUnrollMaxCount = *UserFullUnrollMaxCount;
return UP;
}
TargetTransformInfo::PeelingPreferences
llvm::gatherPeelingPreferences (Loop *L, ScalarEvolution &SE,
const TargetTransformInfo &TTI,
Optional<bool > UserAllowPeeling,
Optional<bool > UserAllowProfileBasedPeeling) {
TargetTransformInfo::PeelingPreferences PP;
// Get Target Specifc Values
TTI.getPeelingPreferences (L, SE, PP);
// User Specified Values using cl::opt
if (UnrollPeelCount.getNumOccurrences () > 0 )
PP.PeelCount = UnrollPeelCount;
if (UnrollAllowPeeling.getNumOccurrences () > 0 )
PP.AllowPeeling = UnrollAllowPeeling;
if (UnrollAllowLoopNestsPeeling.getNumOccurrences () > 0 )
PP.AllowLoopNestsPeeling = UnrollAllowLoopNestsPeeling;
// User Specifed values provided by argument
if (UserAllowPeeling.hasValue ())
PP.AllowPeeling = *UserAllowPeeling;
if (UserAllowProfileBasedPeeling.hasValue ())
PP.PeelProfiledIterations = *UserAllowProfileBasedPeeling;
return PP;
}
namespace {
// / A struct to densely store the state of an instruction after unrolling at
Expand Down
Expand Up
@@ -772,8 +761,7 @@ bool llvm::computeUnrollCount(
ScalarEvolution &SE, const SmallPtrSetImpl<const Value *> &EphValues,
OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount,
bool MaxOrZero, unsigned &TripMultiple, unsigned LoopSize,
TargetTransformInfo::UnrollingPreferences &UP,
TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound) {
TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound) {
// Check for explicit Count.
// 1st priority is unroll count set by "unroll-count" option.
Expand Down
Expand Up
@@ -875,8 +863,8 @@ bool llvm::computeUnrollCount(
}
// 4th priority is loop peeling.
computePeelCount (L, LoopSize, UP, PP, TripCount, SE);
if (PP .PeelCount ) {
computePeelCount (L, LoopSize, UP, TripCount, SE);
if (UP .PeelCount ) {
UP.Runtime = false ;
UP.Count = 1 ;
return ExplicitUnroll;
Expand Down
Expand Up
@@ -1079,9 +1067,8 @@ static LoopUnrollResult tryToUnrollLoop(
TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences (
L, SE, TTI, BFI, PSI, OptLevel, ProvidedThreshold, ProvidedCount,
ProvidedAllowPartial, ProvidedRuntime, ProvidedUpperBound,
ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling,
ProvidedFullUnrollMaxCount);
TargetTransformInfo::PeelingPreferences PP = gatherPeelingPreferences (
L, SE, TTI, ProvidedAllowPeeling, ProvidedAllowProfileBasedPeeling);
// Exit early if unrolling is disabled. For OptForSize, we pick the loop size
// as threshold later on.
Expand Down
Expand Up
@@ -1155,7 +1142,7 @@ static LoopUnrollResult tryToUnrollLoop(
bool UseUpperBound = false ;
bool IsCountSetExplicitly = computeUnrollCount (
L, TTI, DT, LI, SE, EphValues, &ORE, TripCount, MaxTripCount, MaxOrZero,
TripMultiple, LoopSize, UP, PP, UseUpperBound);
TripMultiple, LoopSize, UP, UseUpperBound);
if (!UP.Count )
return LoopUnrollResult::Unmodified;
// Unroll factor (Count) must be less or equal to TripCount.
Expand All
@@ -1170,7 +1157,7 @@ static LoopUnrollResult tryToUnrollLoop(
LoopUnrollResult UnrollResult = UnrollLoop (
L,
{UP.Count , TripCount, UP.Force , UP.Runtime , UP.AllowExpensiveTripCount ,
UseUpperBound, MaxOrZero, TripMultiple, PP .PeelCount , UP.UnrollRemainder ,
UseUpperBound, MaxOrZero, TripMultiple, UP .PeelCount , UP.UnrollRemainder ,
ForgetAllSCEV},
LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop);
if (UnrollResult == LoopUnrollResult::Unmodified)
Expand Down
Expand Up
@@ -1202,7 +1189,7 @@ static LoopUnrollResult tryToUnrollLoop(
// If the loop was peeled, we already "used up" the profile information
// we had, so we don't want to unroll or peel again.
if (UnrollResult != LoopUnrollResult::FullyUnrolled &&
(IsCountSetExplicitly || (PP .PeelProfiledIterations && PP .PeelCount )))
(IsCountSetExplicitly || (UP .PeelProfiledIterations && UP .PeelCount )))
L->setLoopAlreadyUnrolled ();
return UnrollResult;
Expand Down