diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index 47127a6c29603..25703dd6b61f0 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -1293,19 +1293,19 @@ class PostGenericScheduler : public GenericSchedulerBase { protected: ScheduleDAGMI *DAG = nullptr; SchedBoundary Top; - SmallVector BotRoots; + SchedBoundary Bot; + MachineSchedPolicy RegionPolicy; public: - PostGenericScheduler(const MachineSchedContext *C): - GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ") {} + PostGenericScheduler(const MachineSchedContext *C) + : GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ"), + Bot(SchedBoundary::BotQID, "BotQ") {} ~PostGenericScheduler() override = default; void initPolicy(MachineBasicBlock::iterator Begin, MachineBasicBlock::iterator End, - unsigned NumRegionInstrs) override { - /* no configurable policy */ - } + unsigned NumRegionInstrs) override; /// PostRA scheduling does not track pressure. bool shouldTrackPressure() const override { return false; } @@ -1328,15 +1328,16 @@ class PostGenericScheduler : public GenericSchedulerBase { Top.releaseNode(SU, SU->TopReadyCycle, false); } - // Only called for roots. void releaseBottomNode(SUnit *SU) override { - BotRoots.push_back(SU); + if (SU->isScheduled) + return; + Bot.releaseNode(SU, SU->BotReadyCycle, false); } protected: virtual bool tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand); - void pickNodeFromQueue(SchedCandidate &Cand); + void pickNodeFromQueue(SchedBoundary &Zone, SchedCandidate &Cand); }; /// Create the standard converging machine scheduler. This will be used as the diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index ef7662a8e7a26..85de18f5169e5 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -191,7 +191,19 @@ namespace llvm { /// applicable). using SUList = std::list; + /// The direction that should be used to dump the scheduled Sequence. + enum DumpDirection { + TopDown, + BottomUp, + Bidirectional, + NotSet, + }; + + void setDumpDirection(DumpDirection D) { DumpDir = D; } + protected: + DumpDirection DumpDir = NotSet; + /// A map from ValueType to SUList, used during DAG construction, as /// a means of remembering which SUs depend on which memory locations. class Value2SUsMap; diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index 750f739a55990..3bbd126bdaf1a 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -81,6 +81,22 @@ cl::opt ForceTopDown("misched-topdown", cl::Hidden, cl::desc("Force top-down list scheduling")); cl::opt ForceBottomUp("misched-bottomup", cl::Hidden, cl::desc("Force bottom-up list scheduling")); +namespace MISchedPostRASched { +enum Direction { + TopDown, + BottomUp, +}; +} // end namespace MISchedPostRASched +cl::opt PostRADirection( + "misched-postra-direction", cl::Hidden, + cl::desc("Post reg-alloc list scheduling direction"), + // Default to top-down because it was implemented first and existing targets + // expect that behavior by default. + cl::init(MISchedPostRASched::TopDown), + cl::values(clEnumValN(MISchedPostRASched::TopDown, "topdown", + "Force top-down post reg-alloc list scheduling"), + clEnumValN(MISchedPostRASched::BottomUp, "bottomup", + "Force bottom-up post reg-alloc list scheduling"))); cl::opt DumpCriticalPathLength("misched-dcpl", cl::Hidden, cl::desc("Print critical path length to stdout")); @@ -440,6 +456,14 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { // Instantiate the selected scheduler for this target, function, and // optimization level. std::unique_ptr Scheduler(createMachineScheduler()); + ScheduleDAGMI::DumpDirection D; + if (ForceTopDown) + D = ScheduleDAGMI::DumpDirection::TopDown; + else if (ForceBottomUp) + D = ScheduleDAGMI::DumpDirection::BottomUp; + else + D = ScheduleDAGMI::DumpDirection::Bidirectional; + Scheduler->setDumpDirection(D); scheduleRegions(*Scheduler, false); LLVM_DEBUG(LIS->dump()); @@ -473,6 +497,12 @@ bool PostMachineScheduler::runOnMachineFunction(MachineFunction &mf) { // Instantiate the selected scheduler for this target, function, and // optimization level. std::unique_ptr Scheduler(createPostMachineScheduler()); + ScheduleDAGMI::DumpDirection D; + if (PostRADirection == MISchedPostRASched::TopDown) + D = ScheduleDAGMI::DumpDirection::TopDown; + else + D = ScheduleDAGMI::DumpDirection::BottomUp; + Scheduler->setDumpDirection(D); scheduleRegions(*Scheduler, true); if (VerifyScheduling) @@ -1125,12 +1155,14 @@ LLVM_DUMP_METHOD void ScheduleDAGMI::dumpScheduleTraceBottomUp() const { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void ScheduleDAGMI::dumpSchedule() const { if (MISchedDumpScheduleTrace) { - if (ForceTopDown) + if (DumpDir == DumpDirection::TopDown) dumpScheduleTraceTopDown(); - else if (ForceBottomUp) + else if (DumpDir == DumpDirection::BottomUp) dumpScheduleTraceBottomUp(); - else { + else if (DumpDir == DumpDirection::Bidirectional) { dbgs() << "* Schedule table (Bidirectional): not implemented\n"; + } else { + dbgs() << "* Schedule table: DumpDirection not set.\n"; } } @@ -3832,7 +3864,7 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) { Rem.init(DAG, SchedModel); Top.init(DAG, SchedModel, &Rem); - BotRoots.clear(); + Bot.init(DAG, SchedModel, &Rem); // Initialize the HazardRecognizers. If itineraries don't exist, are empty, // or are disabled, then these HazardRecs will be disabled. @@ -3842,13 +3874,30 @@ void PostGenericScheduler::initialize(ScheduleDAGMI *Dag) { DAG->MF.getSubtarget().getInstrInfo()->CreateTargetMIHazardRecognizer( Itin, DAG); } + if (!Bot.HazardRec) { + Bot.HazardRec = + DAG->MF.getSubtarget().getInstrInfo()->CreateTargetMIHazardRecognizer( + Itin, DAG); + } +} + +void PostGenericScheduler::initPolicy(MachineBasicBlock::iterator Begin, + MachineBasicBlock::iterator End, + unsigned NumRegionInstrs) { + if (PostRADirection == MISchedPostRASched::TopDown) { + RegionPolicy.OnlyTopDown = true; + RegionPolicy.OnlyBottomUp = false; + } else if (PostRADirection == MISchedPostRASched::BottomUp) { + RegionPolicy.OnlyTopDown = false; + RegionPolicy.OnlyBottomUp = true; + } } void PostGenericScheduler::registerRoots() { Rem.CriticalPath = DAG->ExitSU.getDepth(); // Some roots may not feed into ExitSU. Check all of them in case. - for (const SUnit *SU : BotRoots) { + for (const SUnit *SU : Bot.Available) { if (SU->getDepth() > Rem.CriticalPath) Rem.CriticalPath = SU->getDepth(); } @@ -3905,12 +3954,13 @@ bool PostGenericScheduler::tryCandidate(SchedCandidate &Cand, return false; } -void PostGenericScheduler::pickNodeFromQueue(SchedCandidate &Cand) { - ReadyQueue &Q = Top.Available; +void PostGenericScheduler::pickNodeFromQueue(SchedBoundary &Zone, + SchedCandidate &Cand) { + ReadyQueue &Q = Zone.Available; for (SUnit *SU : Q) { SchedCandidate TryCand(Cand.Policy); TryCand.SU = SU; - TryCand.AtTop = true; + TryCand.AtTop = Zone.isTop(); TryCand.initResourceDelta(DAG, SchedModel); if (tryCandidate(Cand, TryCand)) { Cand.setBest(TryCand); @@ -3922,29 +3972,54 @@ void PostGenericScheduler::pickNodeFromQueue(SchedCandidate &Cand) { /// Pick the next node to schedule. SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) { if (DAG->top() == DAG->bottom()) { - assert(Top.Available.empty() && Top.Pending.empty() && "ReadyQ garbage"); + assert(Top.Available.empty() && Top.Pending.empty() && + Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage"); return nullptr; } SUnit *SU; do { - SU = Top.pickOnlyChoice(); - if (SU) { - tracePick(Only1, true); + if (RegionPolicy.OnlyBottomUp) { + assert(!RegionPolicy.OnlyTopDown); + SU = Bot.pickOnlyChoice(); + if (SU) { + tracePick(Only1, true); + } else { + CandPolicy NoPolicy; + SchedCandidate BotCand(NoPolicy); + // Set the bottom-up policy based on the state of the current bottom + // zone and the instructions outside the zone, including the top zone. + setPolicy(BotCand.Policy, /*IsPostRA=*/true, Bot, nullptr); + pickNodeFromQueue(Bot, BotCand); + assert(BotCand.Reason != NoCand && "failed to find a candidate"); + tracePick(BotCand); + SU = BotCand.SU; + } + IsTopNode = false; } else { - CandPolicy NoPolicy; - SchedCandidate TopCand(NoPolicy); - // Set the top-down policy based on the state of the current top zone and - // the instructions outside the zone, including the bottom zone. - setPolicy(TopCand.Policy, /*IsPostRA=*/true, Top, nullptr); - pickNodeFromQueue(TopCand); - assert(TopCand.Reason != NoCand && "failed to find a candidate"); - tracePick(TopCand); - SU = TopCand.SU; + + assert(RegionPolicy.OnlyTopDown); + SU = Top.pickOnlyChoice(); + if (SU) { + tracePick(Only1, true); + } else { + CandPolicy NoPolicy; + SchedCandidate TopCand(NoPolicy); + // Set the top-down policy based on the state of the current top zone + // and the instructions outside the zone, including the bottom zone. + setPolicy(TopCand.Policy, /*IsPostRA=*/true, Top, nullptr); + pickNodeFromQueue(Top, TopCand); + assert(TopCand.Reason != NoCand && "failed to find a candidate"); + tracePick(TopCand); + SU = TopCand.SU; + } + IsTopNode = true; } } while (SU->isScheduled); - IsTopNode = true; - Top.removeReady(SU); + if (SU->isTopReady()) + Top.removeReady(SU); + if (SU->isBottomReady()) + Bot.removeReady(SU); LLVM_DEBUG(dbgs() << "Scheduling SU(" << SU->NodeNum << ") " << *SU->getInstr()); @@ -3954,8 +4029,13 @@ SUnit *PostGenericScheduler::pickNode(bool &IsTopNode) { /// Called after ScheduleDAGMI has scheduled an instruction and updated /// scheduled/remaining flags in the DAG nodes. void PostGenericScheduler::schedNode(SUnit *SU, bool IsTopNode) { - SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle()); - Top.bumpNode(SU); + if (IsTopNode) { + SU->TopReadyCycle = std::max(SU->TopReadyCycle, Top.getCurrCycle()); + Top.bumpNode(SU); + } else { + SU->BotReadyCycle = std::max(SU->BotReadyCycle, Bot.getCurrCycle()); + Bot.bumpNode(SU); + } } ScheduleDAGMI *llvm::createGenericSchedPostRA(MachineSchedContext *C) { diff --git a/llvm/test/CodeGen/RISCV/misched-postra-direction.mir b/llvm/test/CodeGen/RISCV/misched-postra-direction.mir new file mode 100644 index 0000000000000..841d0e6d65da6 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/misched-postra-direction.mir @@ -0,0 +1,53 @@ +# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=topdown -o - %s 2>&1 | FileCheck --check-prefix=TOPDOWN %s +# RUN: llc -mtriple=riscv64 -mcpu=sifive-x280 -run-pass=postmisched -enable-post-misched -debug-only=machine-scheduler -misched-dump-schedule-trace -misched-postra-direction=bottomup -o - %s 2>&1 | FileCheck --check-prefix=BOTTOMUP %s + +# REQUIRES: asserts + +--- +name: test +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10, $x11 + + renamable $x12 = MUL renamable $x11, renamable $x10 + renamable $x13 = ADD renamable $x11, renamable $x10 + renamable $x14 = DIVW killed renamable $x12, killed renamable $x13 + PseudoRET implicit $x14 +... + +# TOPDOWN: *** Final schedule for %bb.0 *** +# TOPDOWN-NEXT: * Schedule table (TopDown): +# TOPDOWN-NEXT: i: issue +# TOPDOWN-NEXT: x: resource booked +# TOPDOWN-NEXT: Cycle | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | +# TOPDOWN-NEXT: SU(0) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SiFive7PipeB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SU(1) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SU(2) | | | | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SiFive7PipeAB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SiFive7PipeB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# TOPDOWN-NEXT: SiFive7IDiv | | | | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | +# TOPDOWN-NEXT: SU(0): renamable $x12 = MUL renamable $x11, renamable $x10 +# TOPDOWN-NEXT: SU(1): renamable $x13 = ADD renamable $x11, renamable $x10 +# TOPDOWN-NEXT: SU(2): renamable $x14 = DIVW renamable $x12, renamable $x13 + +# BOTTOMUP: *** Final schedule for %bb.0 *** +# BOTTOMUP-NEXT: * Schedule table (BottomUp): +# BOTTOMUP-NEXT: i: issue +# BOTTOMUP-NEXT: x: resource booked +# BOTTOMUP-NEXT: Cycle | 37 | 36 | 35 | 34 | 33 | 32 | 31 | 30 | 29 | 28 | 27 | 26 | 25 | 24 | 23 | 22 | 21 | 20 | 19 | 18 | 17 | 16 | 15 | 14 | 13 | 12 | 11 | 10 | 9 | 8 | 7 | 6 | 5 | 4 | 3 | 2 | +# BOTTOMUP-NEXT: SU(1) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SU(0) | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SiFive7PipeAB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SiFive7PipeB | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SU(2) | | | | i | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SiFive7PipeAB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SiFive7PipeB | | | | x | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | +# BOTTOMUP-NEXT: SiFive7IDiv | | | | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | x | +# BOTTOMUP-NEXT: SU(1): renamable $x13 = ADD renamable $x11, renamable $x10 +# BOTTOMUP-NEXT: SU(0): renamable $x12 = MUL renamable $x11, renamable $x10 +# BOTTOMUP-NEXT: SU(2): renamable $x14 = DIVW renamable $x12, renamable $x13