Skip to content

Commit

Permalink
[CodeGen][MISched] Add misched post-regalloc bottom-up scheduling
Browse files Browse the repository at this point in the history 
There is the possibility that the bottom-up direction will lead to
performance improvements on certain targets, as this is certainly the case for
the pre-regalloc GenericScheduler. This patch will give people the
opportunity to experiment for their sub-targets. However, this patch
keeps the top-down approach as the default for the PostGenericScheduler
since that is what subtargets expect today.
  • Loading branch information
@michaelmaitland
michaelmaitland committed Feb 27, 2024
1 parent f7cf1f6 commit 9106b58
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 42 deletions.
19 changes: 10 additions & 9 deletions llvm/include/llvm/CodeGen/MachineScheduler.h
View file
Expand Up @@ -1293,19 +1293,19 @@ class PostGenericScheduler : public GenericSchedulerBase {
protected:
ScheduleDAGMI *DAG = nullptr;
SchedBoundary Top;
SmallVector<SUnit*, 8> BotRoots;
SchedBoundary Bot;
MachineSchedPolicy RegionPolicy;

public:
PostGenericScheduler(const MachineSchedContext *C):
GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ") {}
PostGenericScheduler(const MachineSchedContext *C)
: GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ"),
Bot(SchedBoundary::BotQID, "BotQ") {}

~PostGenericScheduler() override = default;

void initPolicy(MachineBasicBlock::iterator Begin,
MachineBasicBlock::iterator End,
unsigned NumRegionInstrs) override {
/* no configurable policy */
}
unsigned NumRegionInstrs) override;

/// PostRA scheduling does not track pressure.
bool shouldTrackPressure() const override { return false; }
Expand All @@ -1328,15 +1328,16 @@ class PostGenericScheduler : public GenericSchedulerBase {
Top.releaseNode(SU, SU->TopReadyCycle, false);
}

// Only called for roots.
void releaseBottomNode(SUnit *SU) override {
BotRoots.push_back(SU);
if (SU->isScheduled)
return;
Bot.releaseNode(SU, SU->BotReadyCycle, false);
}

protected:
virtual bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand);

void pickNodeFromQueue(SchedCandidate &Cand);
void pickNodeFromQueue(SchedBoundary &Zone, SchedCandidate &Cand);
};

/// Create the standard converging machine scheduler. This will be used as the
Expand Down
128 changes: 95 additions & 33 deletions llvm/lib/CodeGen/MachineScheduler.cpp
View file
Expand Up @@ -81,6 +81,22 @@ cl::opt<bool> ForceTopDown("misched-topdown", cl::Hidden,
cl::desc("Force top-down list scheduling"));
cl::opt<bool> ForceBottomUp("misched-bottomup", cl::Hidden,
cl::desc("Force bottom-up list scheduling"));
namespace MISchedPostRASched {
enum Direction {
TopDown,
BottomUp,
};
} // end namespace MISchedPostRASched
cl::opt<MISchedPostRASched::Direction> PostRADirection(
"misched-postra-direction", cl::Hidden,
cl::desc("Post reg-alloc list scheduling direction"),
// Default to top-down because it was implemented first and existing targets
// expect that behavior by default.
cl::init(MISchedPostRASched::TopDown),
cl::values(clEnumValN(MISchedPostRASched::TopDown, "topdown",
"Force top-down post reg-alloc list scheduling"),
clEnumValN(MISchedPostRASched::BottomUp, "bottomup",
"Force bottom-up post reg-alloc list scheduling")));
cl::opt<bool>
DumpCriticalPathLength("misched-dcpl", cl::Hidden,
cl::desc("Print critical path length to stdout"));
Expand Down Expand Up @@ -440,14 +456,14 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) {
// Instantiate the selected scheduler for this target, function, and
// optimization level.
std::unique_ptr<ScheduleDAGInstrs> Scheduler(createMachineScheduler());
ScheduleDAGMI::DumpDirection Dir;
ScheduleDAGMI::DumpDirection D;
if (ForceTopDown)
Dir = ScheduleDAGMI::DumpDirection::TopDown;
D = ScheduleDAGMI::DumpDirection::TopDown;
else if (ForceBottomUp)
Dir = ScheduleDAGMI::DumpDirection::BottomUp;
D = ScheduleDAGMI::DumpDirection::BottomUp;
else
Dir = ScheduleDAGMI::DumpDirection::Bidirectional;
Scheduler->setDumpDirection(Dir);
D = ScheduleDAGMI::DumpDirection::Bidirectional;
Scheduler->setDumpDirection(D);
scheduleRegions(*Scheduler, false);

LLVM_DEBUG(LIS->dump());
Expand Down Expand Up @@ -481,9 +497,12 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) {
// Instantiate the selected scheduler for this target, function, and
// optimization level.
std::unique_ptr<ScheduleDAGInstrs> Scheduler(createPostMachineScheduler());
Scheduler->setDumpDirection(PostRADirection == MISchedPostRASched::TopDown
? ScheduleDAGMI::DumpDirection::TopDown
: ScheduleDAGMI::DumpDirection::BottomUp);
ScheduleDAGMI::DumpDirection D;
if (PostRADirection == MISchedPostRASched::TopDown)
D = ScheduleDAGMI::DumpDirection::TopDown;
else
D = ScheduleDAGMI::DumpDirection::BottomUp;
Scheduler->setDumpDirection(D);
scheduleRegions(*Scheduler, true);

if (VerifyScheduling)
Expand Down Expand Up @@ -1136,11 +1155,11 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceBottomUp() const {
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
LLVM_DUMP_METHOD void ScheduleDAGMI::dumpSchedule() const {
if (MISchedDumpScheduleTrace) {
if (DumpDir == TopDown)
if (DumpDir == DumpDirection::TopDown)
dumpScheduleTraceTopDown();
else if (DumpDir == BottomUp)
else if (DumpDir == DumpDirection::BottomUp)
dumpScheduleTraceBottomUp();
else if (DumpDir == BottomUp) {
else if (DumpDir == DumpDirection::Bidirectional) {
dbgs() << "* Schedule table (Bidirectional): not implemented\n";
} else {
dbgs() << "* Schedule table: DumpDirection not set.\n";
Expand Down Expand Up @@ -3845,7 +3864,7 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) {

Rem.init(DAG, SchedModel);
Top.init(DAG, SchedModel, &Rem);
BotRoots.clear();
Bot.init(DAG, SchedModel, &Rem);

// Initialize the HazardRecognizers. If itineraries don't exist, are empty,
// or are disabled, then these HazardRecs will be disabled.
Expand All @@ -3862,11 +3881,23 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) {
}
}

void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin,
MachineBasicBlock::iterator End,
unsigned NumRegionInstrs) {
if (PostRADirection == MISchedPostRASched::TopDown) {
RegionPolicy.OnlyTopDown = true;
RegionPolicy.OnlyBottomUp = false;
} else if (PostRADirection == MISchedPostRASched::BottomUp) {
RegionPolicy.OnlyTopDown = false;
RegionPolicy.OnlyBottomUp = true;
}
}

void PostGenericScheduler::registerRoots() {
Rem.CriticalPath = DAG->ExitSU.getDepth();

// Some roots may not feed into ExitSU. Check all of them in case.
for (const SUnit *SU : BotRoots) {
for (const SUnit *SU : Bot.Available) {
if (SU->getDepth() > Rem.CriticalPath)
Rem.CriticalPath = SU->getDepth();
}
Expand Down Expand Up @@ -3923,12 +3954,13 @@ bool PostGenericScheduler::tryCandidate(SchedCandidate &Cand,
return false;
}

void PostGenericScheduler::pickNodeFromQueue(SchedCandidate &Cand) {
ReadyQueue &Q = Top.Available;
void PostGenericScheduler::pickNodeFromQueue(SchedBoundary &Zone,
SchedCandidate &Cand) {
ReadyQueue &Q = Zone.Available;
for (SUnit *SU : Q) {
SchedCandidate TryCand(Cand.Policy);
TryCand.SU = SU;
TryCand.AtTop = true;
TryCand.AtTop = Zone.isTop();
TryCand.initResourceDelta(DAG, SchedModel);
if (tryCandidate(Cand, TryCand)) {
Cand.setBest(TryCand);
Expand All @@ -3940,29 +3972,54 @@ void PostGenericScheduler::pickNodeFromQueue(SchedCandidate &Cand) {
/// Pick the next node to schedule.
SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
if (DAG->top() == DAG->bottom()) {
assert(Top.Available.empty() && Top.Pending.empty() && "ReadyQ garbage");
assert(Top.Available.empty() && Top.Pending.empty() &&
Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
return nullptr;
}
SUnit *SU;
do {
SU = Top.pickOnlyChoice();
if (SU) {
tracePick(Only1, true);
if (RegionPolicy.OnlyBottomUp) {
assert(!RegionPolicy.OnlyTopDown);
SU = Bot.pickOnlyChoice();
if (SU) {
tracePick(Only1, true);
} else {
CandPolicy NoPolicy;
SchedCandidate BotCand(NoPolicy);
// Set the bottom-up policy based on the state of the current bottom
// zone and the instructions outside the zone, including the top zone.
setPolicy(BotCand.Policy, /*IsPostRA=*/true, Bot, nullptr);
pickNodeFromQueue(Bot, BotCand);
assert(BotCand.Reason != NoCand && "failed to find a candidate");
tracePick(BotCand);
SU = BotCand.SU;
}
IsTopNode = false;
} else {
CandPolicy NoPolicy;
SchedCandidate TopCand(NoPolicy);
// Set the top-down policy based on the state of the current top zone and
// the instructions outside the zone, including the bottom zone.
setPolicy(TopCand.Policy, /*IsPostRA=*/true, Top, nullptr);
pickNodeFromQueue(TopCand);
assert(TopCand.Reason != NoCand && "failed to find a candidate");
tracePick(TopCand);
SU = TopCand.SU;

assert(RegionPolicy.OnlyTopDown);
SU = Top.pickOnlyChoice();
if (SU) {
tracePick(Only1, true);
} else {
CandPolicy NoPolicy;
SchedCandidate TopCand(NoPolicy);
// Set the top-down policy based on the state of the current top zone
// and the instructions outside the zone, including the bottom zone.
setPolicy(TopCand.Policy, /*IsPostRA=*/true, Top, nullptr);
pickNodeFromQueue(Top, TopCand);
assert(TopCand.Reason != NoCand && "failed to find a candidate");
tracePick(TopCand);
SU = TopCand.SU;
}
IsTopNode = true;
}
} while (SU->isScheduled);

IsTopNode = true;
Top.removeReady(SU);
if (SU->isTopReady())
Top.removeReady(SU);
if (SU->isBottomReady())
Bot.removeReady(SU);

LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") "
<< *SU->getInstr());
Expand All @@ -3972,8 +4029,13 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) {
/// Called after ScheduleDAGMI has scheduled an instruction and updated
/// scheduled/remaining flags in the DAG nodes.
void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) {
SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
Top.bumpNode(SU);
if (IsTopNode) {
SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle());
Top.bumpNode(SU);
} else {
SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle());
Bot.bumpNode(SU);
}
}

ScheduleDAGMI *llvm::createGenericSchedPostRA(MachineSchedContext *C) {
Expand Down
53 changes: 53 additions & 0 deletions llvm/test/CodeGen/RISCV/misched-postra-direction.mir
View file
@@ -0,0 +1,53 @@
# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=topdown -o - %s 2>&1 | FileCheck --check-prefix=TOPDOWN %s
# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=bottomup -o - %s 2>&1 | FileCheck --check-prefix=BOTTOMUP %s

# REQUIRES: asserts

---
name: test
tracksRegLiveness: true
body: |
bb.0.entry:
liveins: $x10, $x11
renamable $x12 = MUL renamable $x11, renamable $x10
renamable $x13 = ADD renamable $x11, renamable $x10
renamable $x14 = DIVW killed renamable $x12, killed renamable $x13
PseudoRET implicit $x14
...

# TOPDOWN: *** Final schedule for %bb.0 ***
# TOPDOWN-NEXT: * Schedule table (TopDown):
# TOPDOWN-NEXT: i: issue
# TOPDOWN-NEXT: x: resource booked
# TOPDOWN-NEXT: Cycle | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 |
# TOPDOWN-NEXT: SU(0) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SiFive7PipeB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SU(1) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SU(2) | | | | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SiFive7PipeAB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SiFive7PipeB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# TOPDOWN-NEXT: SiFive7IDiv | | | | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x |
# TOPDOWN-NEXT: SU(0): renamable $x12 = MUL renamable $x11, renamable $x10
# TOPDOWN-NEXT: SU(1): renamable $x13 = ADD renamable $x11, renamable $x10
# TOPDOWN-NEXT: SU(2): renamable $x14 = DIVW renamable $x12, renamable $x13

# BOTTOMUP: *** Final schedule for %bb.0 ***
# BOTTOMUP-NEXT: * Schedule table (BottomUp):
# BOTTOMUP-NEXT: i: issue
# BOTTOMUP-NEXT: x: resource booked
# BOTTOMUP-NEXT: Cycle | 37 | 36 | 35 | 34 | 33 | 32 | 31 | 30 | 29 | 28 | 27 | 26 | 25 | 24 | 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 | 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | 7 | 6 | 5 | 4 | 3 | 2 |
# BOTTOMUP-NEXT: SU(1) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SU(0) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SiFive7PipeB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SU(2) | | | | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SiFive7PipeAB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SiFive7PipeB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |
# BOTTOMUP-NEXT: SiFive7IDiv | | | | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x |
# BOTTOMUP-NEXT: SU(1): renamable $x13 = ADD renamable $x11, renamable $x10
# BOTTOMUP-NEXT: SU(0): renamable $x12 = MUL renamable $x11, renamable $x10
# BOTTOMUP-NEXT: SU(2): renamable $x14 = DIVW renamable $x12, renamable $x13

0 comments on commit 9106b58

Please sign in to comment.