diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h index 7966c53ed66745..41dc5c4fd81d1c 100644 --- a/llvm/include/llvm/CodeGen/AsmPrinter.h +++ b/llvm/include/llvm/CodeGen/AsmPrinter.h @@ -141,10 +141,6 @@ class AsmPrinter : public MachineFunctionPass { MCSymbol *CurrentFnEnd = nullptr; MCSymbol *CurExceptionSym = nullptr; - // The symbol used to represent the start of the current BB section of the - // function. This is used to calculate the size of the BB section. - MCSymbol *CurrentSectionBeginSym = nullptr; - // The garbage collection metadata printer table. void *GCMetadataPrinters = nullptr; // Really a DenseMap. diff --git a/llvm/include/llvm/CodeGen/MachineBasicBlock.h b/llvm/include/llvm/CodeGen/MachineBasicBlock.h index 50734260c112f1..e8dabc3eef3985 100644 --- a/llvm/include/llvm/CodeGen/MachineBasicBlock.h +++ b/llvm/include/llvm/CodeGen/MachineBasicBlock.h @@ -46,35 +46,17 @@ class raw_ostream; class TargetRegisterClass; class TargetRegisterInfo; -// This structure uniquely identifies a basic block section. -// Possible values are -// {Type: Default, Number: (unsigned)} (These are regular section IDs) -// {Type: Exception, Number: 0} (ExceptionSectionID) -// {Type: Cold, Number: 0} (ColdSectionID) -struct MBBSectionID { - enum SectionType { - Default = 0, // Regular section (these sections are distinguished by the - // Number field). - Exception, // Special section type for exception handling blocks - Cold, // Special section type for cold blocks - } Type; - unsigned Number; - - MBBSectionID(unsigned N) : Type(Default), Number(N) {} - - // Special unique sections for cold and exception blocks. - const static MBBSectionID ColdSectionID; - const static MBBSectionID ExceptionSectionID; - - bool operator==(const MBBSectionID &Other) const { - return Type == Other.Type && Number == Other.Number; - } - - bool operator!=(const MBBSectionID &Other) const { return !(*this == Other); } - -private: - // This is only used to construct the special cold and exception sections. - MBBSectionID(SectionType T) : Type(T), Number(0) {} +enum MachineBasicBlockSection : unsigned { + /// This is also the order of sections in a function. Basic blocks that are + /// part of the original function section (entry block) come first, followed + /// by exception handling basic blocks, cold basic blocks and finally basic + // blocks that need unique sections. + MBBS_Entry, + MBBS_Exception, + MBBS_Cold, + MBBS_Unique, + /// None implies no sections for any basic block, the default. + MBBS_None, }; template <> struct ilist_traits { @@ -161,14 +143,8 @@ class MachineBasicBlock /// Indicate that this basic block is the entry block of a cleanup funclet. bool IsCleanupFuncletEntry = false; - /// With basic block sections, this stores the Section ID of the basic block. - MBBSectionID SectionID{0}; - - // Indicate that this basic block begins a section. - bool IsBeginSection = false; - - // Indicate that this basic block ends a section. - bool IsEndSection = false; + /// Stores the Section type of the basic block with basic block sections. + MachineBasicBlockSection SectionType = MBBS_None; /// Default target of the callbr of a basic block. bool InlineAsmBrDefaultTarget = false; @@ -459,20 +435,16 @@ class MachineBasicBlock void setIsCleanupFuncletEntry(bool V = true) { IsCleanupFuncletEntry = V; } /// Returns true if this block begins any section. - bool isBeginSection() const { return IsBeginSection; } + bool isBeginSection() const; /// Returns true if this block ends any section. - bool isEndSection() const { return IsEndSection; } + bool isEndSection() const; - void setIsBeginSection(bool V = true) { IsBeginSection = V; } + /// Returns the type of section this basic block belongs to. + MachineBasicBlockSection getSectionType() const { return SectionType; } - void setIsEndSection(bool V = true) { IsEndSection = V; } - - /// Returns the section ID of this basic block. - MBBSectionID getSectionID() const { return SectionID; } - - /// Sets the section ID for this basic block. - void setSectionID(MBBSectionID V) { SectionID = V; } + /// Indicate that the basic block belongs to a Section Type. + void setSectionType(MachineBasicBlockSection V) { SectionType = V; } /// Returns true if this is the indirect dest of an INLINEASM_BR. bool isInlineAsmBrIndirectTarget(const MachineBasicBlock *Tgt) const { @@ -513,9 +485,10 @@ class MachineBasicBlock void moveAfter(MachineBasicBlock *NewBefore); /// Returns true if this and MBB belong to the same section. - bool sameSection(const MachineBasicBlock *MBB) const { - return getSectionID() == MBB->getSectionID(); - } + bool sameSection(const MachineBasicBlock *MBB) const; + + /// Returns the basic block that ends the section which contains this one. + const MachineBasicBlock *getSectionEndMBB() const; /// Update the terminator instructions in block to account for changes to the /// layout. If the block previously used a fallthrough, it may now need a @@ -903,6 +876,12 @@ class MachineBasicBlock /// Return the MCSymbol for this basic block. MCSymbol *getSymbol() const; + /// Sets the MCSymbol corresponding to the end of this basic block. + void setEndMCSymbol(MCSymbol *Sym) { EndMCSymbol = Sym; } + + /// Returns the MCSymbol corresponding to the end of this basic block. + MCSymbol *getEndMCSymbol() const { return EndMCSymbol; } + Optional getIrrLoopHeaderWeight() const { return IrrLoopHeaderWeight; } diff --git a/llvm/include/llvm/CodeGen/MachineFunction.h b/llvm/include/llvm/CodeGen/MachineFunction.h index 819c1f9e07b721..35b1c7d6caac9a 100644 --- a/llvm/include/llvm/CodeGen/MachineFunction.h +++ b/llvm/include/llvm/CodeGen/MachineFunction.h @@ -346,6 +346,11 @@ class MachineFunction { /// Section Type for basic blocks, only relevant with basic block sections. BasicBlockSection BBSectionsType = BasicBlockSection::None; + /// With Basic Block Sections, this stores the bb ranges of cold and + /// exception sections. + std::pair ColdSectionRange = {-1, -1}; + std::pair ExceptionSectionRange = {-1, -1}; + /// List of C++ TypeInfo used. std::vector TypeInfos; @@ -503,13 +508,22 @@ class MachineFunction { void setBBSectionsType(BasicBlockSection V) { BBSectionsType = V; } + void setSectionRange(); + + /// Returns true if this basic block number starts a cold or exception + /// section. + bool isSectionStartMBB(int N) const { + return (N == ColdSectionRange.first || N == ExceptionSectionRange.first); + } + + /// Returns true if this basic block ends a cold or exception section. + bool isSectionEndMBB(int N) const { + return (N == ColdSectionRange.second || N == ExceptionSectionRange.second); + } + /// Creates basic block Labels for this function. void createBBLabels(); - /// Assign IsBeginSection IsEndSection fields for basic blocks in this - /// function. - void assignBeginEndSections(); - /// getTarget - Return the target machine this machine code is compiled with const LLVMTargetMachine &getTarget() const { return Target; } diff --git a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index cf913a17767241..5daf11971c60c8 100644 --- a/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/llvm/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -69,6 +69,10 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile { const MachineBasicBlock &MBB, const TargetMachine &TM) const override; + MCSection *getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const override; + bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference, const Function &F) const override; diff --git a/llvm/include/llvm/Target/TargetLoweringObjectFile.h b/llvm/include/llvm/Target/TargetLoweringObjectFile.h index ffe0bfe74df8fc..924558cb27974e 100644 --- a/llvm/include/llvm/Target/TargetLoweringObjectFile.h +++ b/llvm/include/llvm/Target/TargetLoweringObjectFile.h @@ -92,6 +92,10 @@ class TargetLoweringObjectFile : public MCObjectFileInfo { const MachineBasicBlock &MBB, const TargetMachine &TM) const; + virtual MCSection *getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const; + /// Classify the specified global variable into a set of target independent /// categories embodied in SectionKind. static SectionKind getKindForGlobal(const GlobalObject *GO, diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 8583760742ceb2..b7bd6c9ad9f58b 100644 --- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1097,6 +1097,14 @@ void AsmPrinter::emitFunctionBody() { // Print out code for the function. bool HasAnyRealCode = false; int NumInstsInFunction = 0; + bool emitBBSections = MF->hasBBSections(); + MachineBasicBlock *EndOfRegularSectionMBB = nullptr; + if (emitBBSections) { + EndOfRegularSectionMBB = + const_cast(MF->front().getSectionEndMBB()); + assert(EndOfRegularSectionMBB->isEndSection() && + "The MBB at the end of the regular section must end a section"); + } for (auto &MBB : *MF) { // Print a label for the basic block. @@ -1177,41 +1185,17 @@ void AsmPrinter::emitFunctionBody() { } } } - - // We need a temporary symbol for the end of this basic block, if either we - // have BBLabels enabled and we want to emit size directive for the BBs, or - // if this basic blocks marks the end of a section (except the section - // containing the entry basic block as the end symbol for that section is - // CurrentFnEnd). - MCSymbol *CurrentBBEnd = nullptr; - if ((MAI->hasDotTypeDotSizeDirective() && MF->hasBBLabels()) || - (MBB.isEndSection() && !MBB.sameSection(&MF->front()))) { - CurrentBBEnd = OutContext.createTempSymbol(); - OutStreamer->emitLabel(CurrentBBEnd); - } - - // Helper for emitting the size directive associated with a basic block - // symbol. - auto emitELFSizeDirective = [&](MCSymbol *SymForSize) { - assert(CurrentBBEnd && "Basicblock end symbol not set!"); + if (&MBB != EndOfRegularSectionMBB && + (MF->hasBBLabels() || MBB.isEndSection())) { + // Emit size directive for the size of this basic block. Create a symbol + // for the end of the basic block. + MCSymbol *CurrentBBEnd = OutContext.createTempSymbol(); const MCExpr *SizeExp = MCBinaryExpr::createSub( MCSymbolRefExpr::create(CurrentBBEnd, OutContext), - MCSymbolRefExpr::create(SymForSize, OutContext), OutContext); - OutStreamer->emitELFSize(SymForSize, SizeExp); - }; - - // Emit size directive for the size of each basic block, if BBLabels is - // enabled. - if (MAI->hasDotTypeDotSizeDirective() && MF->hasBBLabels()) - emitELFSizeDirective(MBB.getSymbol()); - - // Emit size directive for the size of each basic block section once we - // get to the end of that section. - if (MBB.isEndSection()) { - if (!MBB.sameSection(&MF->front())) { - if (MAI->hasDotTypeDotSizeDirective()) - emitELFSizeDirective(CurrentSectionBeginSym); - } + MCSymbolRefExpr::create(MBB.getSymbol(), OutContext), OutContext); + OutStreamer->emitLabel(CurrentBBEnd); + MBB.setEndMCSymbol(CurrentBBEnd); + OutStreamer->emitELFSize(MBB.getSymbol(), SizeExp); } emitBasicBlockEnd(MBB); } @@ -1246,8 +1230,9 @@ void AsmPrinter::emitFunctionBody() { } } - // Switch to the original section in case basic block sections was used. - OutStreamer->SwitchSection(MF->getSection()); + // Switch to the original section if basic block sections was used. + if (emitBBSections) + OutStreamer->SwitchSection(MF->getSection()); const Function &F = MF->getFunction(); for (const auto &BB : F) { @@ -1264,7 +1249,7 @@ void AsmPrinter::emitFunctionBody() { emitFunctionBodyEnd(); if (needFuncLabelsForEHOrDebugInfo(*MF, MMI) || - MAI->hasDotTypeDotSizeDirective()) { + MAI->hasDotTypeDotSizeDirective() || emitBBSections) { // Create a symbol for the end of function. CurrentFnEnd = createTempSymbol("func_end"); OutStreamer->emitLabel(CurrentFnEnd); @@ -1287,6 +1272,8 @@ void AsmPrinter::emitFunctionBody() { HI.Handler->markFunctionEnd(); } + if (emitBBSections) + EndOfRegularSectionMBB->setEndMCSymbol(CurrentFnEnd); // Print out jump tables referenced by the function. emitJumpTableInfo(); @@ -1766,7 +1753,6 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { CurrentFnSymForSize = CurrentFnSym; CurrentFnBegin = nullptr; - CurrentSectionBeginSym = nullptr; CurExceptionSym = nullptr; bool NeedsLocalForSize = MAI->needsLocalForSize(); if (F.hasFnAttribute("patchable-function-entry") || @@ -2995,6 +2981,7 @@ static void emitBasicBlockLoopComments(const MachineBasicBlock &MBB, /// MachineBasicBlock, an alignment (if present) and a comment describing /// it if appropriate. void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { + bool BBSections = MF->hasBBSections(); // End the previous funclet and start a new one. if (MBB.isEHFuncletEntry()) { for (const HandlerInfo &HI : Handlers) { @@ -3004,9 +2991,11 @@ void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { } // Emit an alignment directive for this block, if needed. - const Align Alignment = MBB.getAlignment(); - if (Alignment != Align(1)) - emitAlignment(Alignment); + if (MBB.pred_empty() || !BBSections) { + const Align Alignment = MBB.getAlignment(); + if (Alignment != Align(1)) + emitAlignment(Alignment); + } // If the block has its address taken, emit any labels that were used to // reference the block. It is possible that there is more than one label @@ -3038,8 +3027,9 @@ void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { emitBasicBlockLoopComments(MBB, MLI, *this); } + bool emitBBLabels = BBSections || MF->hasBBLabels(); if (MBB.pred_empty() || - (!MF->hasBBLabels() && isBlockOnlyReachableByFallthrough(&MBB) && + (!emitBBLabels && isBlockOnlyReachableByFallthrough(&MBB) && !MBB.isEHFuncletEntry() && !MBB.hasLabelMustBeEmitted())) { if (isVerbose()) { // NOTE: Want this comment at start of line, don't emit with AddComment. @@ -3050,12 +3040,23 @@ void AsmPrinter::emitBasicBlockStart(const MachineBasicBlock &MBB) { if (isVerbose() && MBB.hasLabelMustBeEmitted()) { OutStreamer->AddComment("Label of block must be emitted"); } - // Switch to a new section if this basic block must begin a section. - if (MBB.isBeginSection()) { + // With -fbasicblock-sections, a basic block can start a new section. + if (MBB.getSectionType() == MachineBasicBlockSection::MBBS_Exception) { + // Create the exception section for this function. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".eh")); + } else if (MBB.getSectionType() == MachineBasicBlockSection::MBBS_Cold) { + // Create the cold section here. + OutStreamer->SwitchSection( + getObjFileLowering().getNamedSectionForMachineBasicBlock( + MF->getFunction(), MBB, TM, ".unlikely")); + } else if (MBB.isBeginSection() && MBB.isEndSection()) { OutStreamer->SwitchSection( getObjFileLowering().getSectionForMachineBasicBlock(MF->getFunction(), MBB, TM)); - CurrentSectionBeginSym = MBB.getSymbol(); + } else if (BBSections) { + OutStreamer->SwitchSection(MF->getSection()); } OutStreamer->emitLabel(MBB.getSymbol()); } @@ -3089,7 +3090,7 @@ void AsmPrinter::emitVisibility(MCSymbol *Sym, unsigned Visibility, /// the predecessor and this block is a fall-through. bool AsmPrinter:: isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const { - // With BasicBlock Sections, beginning of the section is not a fallthrough. + // With BasicBlock Sections, no block is a fall through. if (MBB->isBeginSection()) return false; diff --git a/llvm/lib/CodeGen/BBSectionsPrepare.cpp b/llvm/lib/CodeGen/BBSectionsPrepare.cpp index eb71a382be184f..6e81801abf06d4 100644 --- a/llvm/lib/CodeGen/BBSectionsPrepare.cpp +++ b/llvm/lib/CodeGen/BBSectionsPrepare.cpp @@ -9,41 +9,47 @@ // BBSectionsPrepare implementation. // // The purpose of this pass is to assign sections to basic blocks when -// -fbasicblock-sections= option is used. Further, with profile information only -// the subset of basic blocks with profiles are placed in separate sections and -// the rest are grouped in a cold section. The exception handling blocks are -// treated specially to ensure they are all in one seciton. +// -fbasicblock-sections= option is used. Exception landing pad blocks are +// specially handled by grouping them in a single section. Further, with +// profile information only the subset of basic blocks with profiles are placed +// in a separate section and the rest are grouped in a cold section. // // Basic Block Sections // ==================== // -// With option, -fbasicblock-sections=list, every function may be split into -// clusters of basic blocks. Every cluster will be emitted into a separate -// section with its basic blocks sequenced in the given order. To get the -// optimized performance, the clusters must form an optimal BB layout for the -// function. Every cluster's section is labeled with a symbol to allow the -// linker to reorder the sections in any arbitrary sequence. A global order of -// these sections would encapsulate the function layout. +// With option, -fbasicblock-sections=, each basic block could be placed in a +// unique ELF text section in the object file along with a symbol labelling the +// basic block. The linker can then order the basic block sections in any +// arbitrary sequence which when done correctly can encapsulate block layout, +// function layout and function splitting optimizations. However, there are a +// couple of challenges to be addressed for this to be feasible: // -// There are a couple of challenges to be addressed: +// 1. The compiler must not allow any implicit fall-through between any two +// adjacent basic blocks as they could be reordered at link time to be +// non-adjacent. In other words, the compiler must make a fall-through +// between adjacent basic blocks explicit by retaining the direct jump +// instruction that jumps to the next basic block. // -// 1. The last basic block of every cluster should not have any implicit -// fallthrough to its next basic block, as it can be reordered by the linker. -// The compiler should make these fallthroughs explicit by adding -// unconditional jumps.. -// -// 2. All inter-cluster branch targets would now need to be resolved by the +// 2. All inter-basic block branch targets would now need to be resolved by the // linker as they cannot be calculated during compile time. This is done // using static relocations. Further, the compiler tries to use short branch // instructions on some ISAs for small branch offsets. This is not possible -// for inter-cluster branches as the offset is not determined at compile -// time, and therefore, long branch instructions have to be used for those. +// with basic block sections as the offset is not determined at compile time, +// and long branch instructions have to be used everywhere. +// +// 3. Each additional section bloats object file sizes by tens of bytes. The +// number of basic blocks can be potentially very large compared to the size +// of functions and can bloat object sizes significantly. Option +// fbasicblock-sections= also takes a file path which can be used to specify +// a subset of basic blocks that needs unique sections to keep the bloats +// small. // -// 3. Debug Information (DebugInfo) and Call Frame Information (CFI) emission +// 4. Debug Information (DebugInfo) and Call Frame Information (CFI) emission // needs special handling with basic block sections. DebugInfo needs to be // emitted with more relocations as basic block sections can break a // function into potentially several disjoint pieces, and CFI needs to be -// emitted per cluster. This also bloats the object file and binary sizes. +// emitted per basic block. This also bloats the object file and binary +// sizes. // // Basic Block Labels // ================== @@ -64,9 +70,7 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineFunction.h" @@ -75,59 +79,34 @@ #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/InitializePasses.h" -#include "llvm/Support/Error.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Target/TargetMachine.h" +#include + using llvm::SmallSet; -using llvm::SmallVector; using llvm::StringMap; using llvm::StringRef; using namespace llvm; namespace { -// This struct represents the cluster information for a machine basic block. -struct BBClusterInfo { - // MachineBasicBlock ID. - unsigned MBBNumber; - // Cluster ID this basic block belongs to. - unsigned ClusterID; - // Position of basic block within the cluster. - unsigned PositionInCluster; -}; - -using ProgramBBClusterInfoMapTy = StringMap>; - class BBSectionsPrepare : public MachineFunctionPass { public: static char ID; - - // This contains the basic-block-sections profile. + StringMap> BBSectionsList; const MemoryBuffer *MBuf = nullptr; - // This encapsulates the BB cluster information for the whole program. - // - // For every function name, it contains the cluster information for (all or - // some of) its basic blocks. The cluster information for every basic block - // includes its cluster ID along with the position of the basic block in that - // cluster. - ProgramBBClusterInfoMapTy ProgramBBClusterInfo; - - // Some functions have alias names. We use this map to find the main alias - // name for which we have mapping in ProgramBBClusterInfo. - StringMap FuncAliasMap; + BBSectionsPrepare() : MachineFunctionPass(ID) { + initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); + } BBSectionsPrepare(const MemoryBuffer *Buf) : MachineFunctionPass(ID), MBuf(Buf) { initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); }; - BBSectionsPrepare() : MachineFunctionPass(ID) { - initializeBBSectionsPreparePass(*PassRegistry::getPassRegistry()); - } - StringRef getPassName() const override { return "Basic Block Sections Analysis"; } @@ -146,181 +125,104 @@ class BBSectionsPrepare : public MachineFunctionPass { char BBSectionsPrepare::ID = 0; INITIALIZE_PASS(BBSectionsPrepare, "bbsections-prepare", - "Prepares for basic block sections, by splitting functions " - "into clusters of basic blocks.", - false, false) + "Determine if a basic block needs a special section", false, + false) -// This function updates and optimizes the branching instructions of every basic -// block in a given function to account for changes in the layout. -static void updateBranches( - MachineFunction &MF, - const SmallVector &PreLayoutFallThroughs) { - const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); - SmallVector Cond; - for (auto &MBB : MF) { - auto NextMBBI = std::next(MBB.getIterator()); - auto *FTMBB = PreLayoutFallThroughs[MBB.getNumber()]; - // If this block had a fallthrough before we need an explicit unconditional - // branch to that block if either - // 1- the block ends a section, which means its next block may be - // reorderd by the linker, or - // 2- the fallthrough block is not adjacent to the block in the new - // order. - if (FTMBB && (MBB.isEndSection() || &*NextMBBI != FTMBB)) - TII->insertUnconditionalBranch(MBB, FTMBB, MBB.findBranchDebugLoc()); - - // We do not optimize branches for machine basic blocks ending sections, as - // their adjacent block might be reordered by the linker. - if (MBB.isEndSection()) - continue; +// This inserts an unconditional branch at the end of MBB to the next basic +// block S if and only if the control-flow implicitly falls through from MBB to +// S and S and MBB belong to different sections. This is necessary with basic +// block sections as MBB and S could be potentially reordered. +static void insertUnconditionalFallthroughBranch(MachineBasicBlock &MBB) { + MachineBasicBlock *Fallthrough = MBB.getFallThrough(); - // It might be possible to optimize branches by flipping the branch - // condition. - Cond.clear(); - MachineBasicBlock *TBB = nullptr, *FBB = nullptr; // For analyzeBranch. - if (TII->analyzeBranch(MBB, TBB, FBB, Cond)) - continue; - MBB.updateTerminator(); - } -} + if (Fallthrough == nullptr) + return; -// This function provides the BBCluster information associated with a function. -// Returns true if a valid association exists and false otherwise. -static bool getBBClusterInfoForFunction( - const MachineFunction &MF, const StringMap FuncAliasMap, - const ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, - std::vector> &V) { - // Get the main alias name for the function. - auto FuncName = MF.getName(); - auto R = FuncAliasMap.find(FuncName); - StringRef AliasName = R == FuncAliasMap.end() ? FuncName : R->second; - - // Find the assoicated cluster information. - auto P = ProgramBBClusterInfo.find(AliasName); - if (P == ProgramBBClusterInfo.end()) - return false; + // If this basic block and the Fallthrough basic block are in the same + // section then do not insert the jump. + if (MBB.sameSection(Fallthrough)) + return; - if (P->second.empty()) { - // This indicates that sections are desired for all basic blocks of this - // function. We clear the BBClusterInfo vector to denote this. - V.clear(); - return true; - } + const TargetInstrInfo *TII = MBB.getParent()->getSubtarget().getInstrInfo(); + SmallVector Cond; + MachineBasicBlock *TBB = nullptr, *FBB = nullptr; - V.resize(MF.getNumBlockIDs()); - for (auto bbClusterInfo : P->second) { - // Bail out if the cluster information contains invalid MBB numbers. - if (bbClusterInfo.MBBNumber >= MF.getNumBlockIDs()) - return false; - V[bbClusterInfo.MBBNumber] = bbClusterInfo; + // If a branch to the fall through block already exists, return. + if (!TII->analyzeBranch(MBB, TBB, FBB, Cond) && + (TBB == Fallthrough || FBB == Fallthrough)) { + return; } - return true; + + Cond.clear(); + DebugLoc DL = MBB.findBranchDebugLoc(); + TII->insertBranch(MBB, Fallthrough, nullptr, Cond, DL); } -// This function sorts basic blocks according to the cluster's information. -// All explicitly specified clusters of basic blocks will be ordered -// accordingly. All non-specified BBs go into a separate "Cold" section. -// Additionally, if exception handling landing pads end up in more than one -// clusters, they are moved into a single "Exception" section. Eventually, -// clusters are ordered in increasing order of their IDs, with the "Exception" -// and "Cold" succeeding all other clusters. -// FuncBBClusterInfo represent the cluster information for basic blocks. If this -// is empty, it means unique sections for all basic blocks in the function. +/// This function sorts basic blocks according to the sections in which they are +/// emitted. Basic block sections automatically turn on function sections so +/// the entry block is in the function section. The other sections that are +/// created are: +/// 1) Exception section - basic blocks that are landing pads +/// 2) Cold section - basic blocks that will not have unique sections. +/// 3) Unique section - one per basic block that is emitted in a unique section. static bool assignSectionsAndSortBasicBlocks( MachineFunction &MF, - const std::vector> &FuncBBClusterInfo) { - assert(MF.hasBBSections() && "BB Sections is not set for function."); - // This variable stores the section ID of the cluster containing eh_pads (if - // all eh_pads are one cluster). If more than one cluster contain eh_pads, we - // set it equal to ExceptionSectionID. - Optional EHPadsSectionID; + const StringMap> &BBSectionsList) { + SmallSet S = BBSectionsList.lookup(MF.getName()); + + bool HasHotEHPads = false; for (auto &MBB : MF) { - // With the 'all' option, every basic block is placed in a unique section. - // With the 'list' option, every basic block is placed in a section - // associated with its cluster, unless we want individual unique sections - // for every basic block in this function (if FuncBBClusterInfo is empty). - if (MF.getTarget().getBBSectionsType() == llvm::BasicBlockSection::All || - FuncBBClusterInfo.empty()) { - // If unique sections are desired for all basic blocks of the function, we - // set every basic block's section ID equal to its number (basic block - // id). This further ensures that basic blocks are ordered canonically. - MBB.setSectionID({static_cast(MBB.getNumber())}); - } else if (FuncBBClusterInfo[MBB.getNumber()].hasValue()) - MBB.setSectionID(FuncBBClusterInfo[MBB.getNumber()]->ClusterID); - else { - // BB goes into the special cold section if it is not specified in the - // cluster info map. - MBB.setSectionID(MBBSectionID::ColdSectionID); + // Entry basic block cannot start another section because the function + // starts one already. + if (MBB.getNumber() == MF.front().getNumber()) { + MBB.setSectionType(MachineBasicBlockSection::MBBS_Entry); + continue; } - - if (MBB.isEHPad() && EHPadsSectionID != MBB.getSectionID() && - EHPadsSectionID != MBBSectionID::ExceptionSectionID) { - // If we already have one cluster containing eh_pads, this must be updated - // to ExceptionSectionID. Otherwise, we set it equal to the current - // section ID. - EHPadsSectionID = EHPadsSectionID.hasValue() - ? MBBSectionID::ExceptionSectionID - : MBB.getSectionID(); + // Check if this BB is a cold basic block. With the list option, all cold + // basic blocks can be clustered in a single cold section. + // All Exception landing pads must be in a single section. If all the + // landing pads are cold, it can be kept in the cold section. Otherwise, we + // create a separate exception section. + bool isColdBB = ((MF.getTarget().getBBSectionsType() == + llvm::BasicBlockSection::List) && + !S.empty() && !S.count(MBB.getNumber())); + if (isColdBB) { + MBB.setSectionType(MachineBasicBlockSection::MBBS_Cold); + } else if (MBB.isEHPad()) { + // We handle non-cold basic eh blocks later. + HasHotEHPads = true; + } else { + // Place this MBB in a unique section. A unique section begins and ends + // that section by definition. + MBB.setSectionType(MachineBasicBlockSection::MBBS_Unique); } } - // If EHPads are in more than one section, this places all of them in the - // special exception section. - if (EHPadsSectionID == MBBSectionID::ExceptionSectionID) - for (auto &MBB : MF) + // If some EH Pads are not cold then we move all EH Pads to the exception + // section as we require that all EH Pads be in a single section. + if (HasHotEHPads) { + std::for_each(MF.begin(), MF.end(), [&](MachineBasicBlock &MBB) { if (MBB.isEHPad()) - MBB.setSectionID(EHPadsSectionID.getValue()); - - SmallVector PreLayoutFallThroughs( - MF.getNumBlockIDs()); - for (auto &MBB : MF) - PreLayoutFallThroughs[MBB.getNumber()] = MBB.getFallThrough(); - - // We make sure that the cluster including the entry basic block precedes all - // other clusters. - auto EntryBBSectionID = MF.front().getSectionID(); - - // Helper function for ordering BB sections as follows: - // * Entry section (section including the entry block). - // * Regular sections (in increasing order of their Number). - // ... - // * Exception section - // * Cold section - auto MBBSectionOrder = [EntryBBSectionID](const MBBSectionID &LHS, - const MBBSectionID &RHS) { - // We make sure that the section containing the entry block precedes all the - // other sections. - if (LHS == EntryBBSectionID || RHS == EntryBBSectionID) - return LHS == EntryBBSectionID; - return LHS.Type == RHS.Type ? LHS.Number < RHS.Number : LHS.Type < RHS.Type; - }; + MBB.setSectionType(MachineBasicBlockSection::MBBS_Exception); + }); + } - // We sort all basic blocks to make sure the basic blocks of every cluster are - // contiguous and ordered accordingly. Furthermore, clusters are ordered in - // increasing order of their section IDs, with the exception and the - // cold section placed at the end of the function. - MF.sort([&](MachineBasicBlock &X, MachineBasicBlock &Y) { - auto XSectionID = X.getSectionID(); - auto YSectionID = Y.getSectionID(); - if (XSectionID != YSectionID) - return MBBSectionOrder(XSectionID, YSectionID); - // If the two basic block are in the same section, the order is decided by - // their position within the section. - if (XSectionID.Type == MBBSectionID::SectionType::Default) - return FuncBBClusterInfo[X.getNumber()]->PositionInCluster < - FuncBBClusterInfo[Y.getNumber()]->PositionInCluster; - return X.getNumber() < Y.getNumber(); - }); - - // Set IsBeginSection and IsEndSection according to the assigned section IDs. - MF.assignBeginEndSections(); - - // After reordering basic blocks, we must update basic block branches to - // insert explicit fallthrough branches when required and optimize branches - // when possible. - updateBranches(MF, PreLayoutFallThroughs); + for (auto &MBB : MF) { + // With -fbasicblock-sections, fall through blocks must be made + // explicitly reachable. Do this after sections is set as + // unnecessary fallthroughs can be avoided. + insertUnconditionalFallthroughBranch(MBB); + } + + MF.sort(([&](MachineBasicBlock &X, MachineBasicBlock &Y) { + unsigned TypeX = X.getSectionType(); + unsigned TypeY = Y.getSectionType(); + return (TypeX != TypeY) ? TypeX < TypeY : X.getNumber() < Y.getNumber(); + })); + + MF.setSectionRange(); return true; } @@ -328,6 +230,7 @@ bool BBSectionsPrepare::runOnMachineFunction(MachineFunction &MF) { auto BBSectionsType = MF.getTarget().getBBSectionsType(); assert(BBSectionsType != BasicBlockSection::None && "BB Sections not enabled!"); + // Renumber blocks before sorting them for basic block sections. This is // useful during sorting, basic blocks in the same section will retain the // default order. This renumbering should also be done for basic block @@ -340,110 +243,65 @@ bool BBSectionsPrepare::runOnMachineFunction(MachineFunction &MF) { return true; } - std::vector> FuncBBClusterInfo; if (BBSectionsType == BasicBlockSection::List && - !getBBClusterInfoForFunction(MF, FuncAliasMap, ProgramBBClusterInfo, - FuncBBClusterInfo)) + BBSectionsList.find(MF.getName()) == BBSectionsList.end()) return true; + MF.setBBSectionsType(BBSectionsType); MF.createBBLabels(); - assignSectionsAndSortBasicBlocks(MF, FuncBBClusterInfo); + assignSectionsAndSortBasicBlocks(MF, BBSectionsList); + return true; } // Basic Block Sections can be enabled for a subset of machine basic blocks. // This is done by passing a file containing names of functions for which basic // block sections are desired. Additionally, machine basic block ids of the -// functions can also be specified for a finer granularity. Moreover, a cluster -// of basic blocks could be assigned to the same section. -// A file with basic block sections for all of function main and three blocks -// for function foo (of which 1 and 2 are placed in a cluster) looks like this: +// functions can also be specified for a finer granularity. +// A file with basic block sections for all of function main and two blocks for +// function foo looks like this: // ---------------------------- // list.txt: // !main // !foo -// !!1 2 +// !!2 // !!4 -static Error getBBClusterInfo(const MemoryBuffer *MBuf, - ProgramBBClusterInfoMapTy &ProgramBBClusterInfo, - StringMap &FuncAliasMap) { - assert(MBuf); - line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); - - auto invalidProfileError = [&](auto Message) { - return make_error( - Twine("Invalid profile " + MBuf->getBufferIdentifier() + " at line " + - Twine(LineIt.line_number()) + ": " + Message), - inconvertibleErrorCode()); - }; - - auto FI = ProgramBBClusterInfo.end(); +static bool getBBSectionsList(const MemoryBuffer *MBuf, + StringMap> &bbMap) { + if (!MBuf) + return false; - // Current cluster ID corresponding to this function. - unsigned CurrentCluster = 0; - // Current position in the current cluster. - unsigned CurrentPosition = 0; + line_iterator LineIt(*MBuf, /*SkipBlanks=*/true, /*CommentMarker=*/'#'); - // Temporary set to ensure every basic block ID appears once in the clusters - // of a function. - SmallSet FuncBBIDs; + StringMap>::iterator fi = bbMap.end(); for (; !LineIt.is_at_eof(); ++LineIt) { - StringRef S(*LineIt); - if (S[0] == '@') + StringRef s(*LineIt); + if (s[0] == '@') continue; // Check for the leading "!" - if (!S.consume_front("!") || S.empty()) + if (!s.consume_front("!") || s.empty()) break; - // Check for second "!" which indicates a cluster of basic blocks. - if (S.consume_front("!")) { - if (FI == ProgramBBClusterInfo.end()) - return invalidProfileError( - "Cluster list does not follow a function name specifier."); - SmallVector BBIndexes; - S.split(BBIndexes, ' '); - // Reset current cluster position. - CurrentPosition = 0; - for (auto BBIndexStr : BBIndexes) { - unsigned long long BBIndex; - if (getAsUnsignedInteger(BBIndexStr, 10, BBIndex)) - return invalidProfileError(Twine("Unsigned integer expected: '") + - BBIndexStr + "'."); - if (!FuncBBIDs.insert(BBIndex).second) - return invalidProfileError(Twine("Duplicate basic block id found '") + - BBIndexStr + "'."); - if (!BBIndex && CurrentPosition) - return invalidProfileError("Entry BB (0) does not begin a cluster."); - - FI->second.emplace_back(BBClusterInfo{ - ((unsigned)BBIndex), CurrentCluster, CurrentPosition++}); - } - CurrentCluster++; - } else { // This is a function name specifier. - // Function aliases are separated using '/'. We use the first function - // name for the cluster info mapping and delegate all other aliases to - // this one. - SmallVector Aliases; - S.split(Aliases, '/'); - for (size_t i = 1; i < Aliases.size(); ++i) - FuncAliasMap.try_emplace(Aliases[i], Aliases.front()); - - // Prepare for parsing clusters of this function name. - // Start a new cluster map for this function name. - FI = ProgramBBClusterInfo.try_emplace(Aliases.front()).first; - CurrentCluster = 0; - FuncBBIDs.clear(); + // Check for second "!" which encodes basic block ids. + if (s.consume_front("!")) { + if (fi != bbMap.end()) + fi->second.insert(std::stoi(s.str())); + else + return false; + } else { + // Start a new function. + auto R = bbMap.try_emplace(s.split('/').first); + fi = R.first; + assert(R.second); } } - return Error::success(); + return true; } bool BBSectionsPrepare::doInitialization(Module &M) { - if (!MBuf) - return false; - if (auto Err = getBBClusterInfo(MBuf, ProgramBBClusterInfo, FuncAliasMap)) - report_fatal_error(std::move(Err)); - return false; + if (MBuf) + getBBSectionsList(MBuf, BBSectionsList); + return true; } void BBSectionsPrepare::getAnalysisUsage(AnalysisUsage &AU) const { diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp index 0107e5e895a2b8..68de9f49fe4885 100644 --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -495,7 +495,7 @@ class MIParser { bool parseOffset(int64_t &Offset); bool parseAlignment(unsigned &Alignment); bool parseAddrspace(unsigned &Addrspace); - bool parseSectionID(Optional &SID); + bool parseMBBS(MachineBasicBlockSection &T); bool parseOperandsOffset(MachineOperand &Op); bool parseIRValue(const Value *&V); bool parseMemoryOperandFlag(MachineMemOperand::Flags &Flags); @@ -620,24 +620,21 @@ bool MIParser::consumeIfPresent(MIToken::TokenKind TokenKind) { return true; } -// Parse Machine Basic Block Section ID. -bool MIParser::parseSectionID(Optional &SID) { +// Parse Machine Basic Block Section Type. +bool MIParser::parseMBBS(MachineBasicBlockSection &T) { assert(Token.is(MIToken::kw_bbsections)); lex(); - if (Token.is(MIToken::IntegerLiteral)) { - unsigned Value = 0; - if (getUnsigned(Value)) - return error("Unknown Section ID"); - SID = MBBSectionID{Value}; - } else { - const StringRef &S = Token.stringValue(); - if (S == "Exception") - SID = MBBSectionID::ExceptionSectionID; - else if (S == "Cold") - SID = MBBSectionID::ColdSectionID; - else - return error("Unknown Section ID"); - } + const StringRef &S = Token.stringValue(); + if (S == "Entry") + T = MBBS_Entry; + else if (S == "Exception") + T = MBBS_Exception; + else if (S == "Cold") + T = MBBS_Cold; + else if (S == "Unique") + T = MBBS_Unique; + else + return error("Unknown Section Type"); lex(); return false; } @@ -654,7 +651,7 @@ bool MIParser::parseBasicBlockDefinition( bool HasAddressTaken = false; bool IsLandingPad = false; bool IsEHFuncletEntry = false; - Optional SectionID; + MachineBasicBlockSection SectionType = MBBS_None; unsigned Alignment = 0; BasicBlock *BB = nullptr; if (consumeIfPresent(MIToken::lparen)) { @@ -684,7 +681,7 @@ bool MIParser::parseBasicBlockDefinition( lex(); break; case MIToken::kw_bbsections: - if (parseSectionID(SectionID)) + if (parseMBBS(SectionType)) return true; break; default: @@ -717,8 +714,8 @@ bool MIParser::parseBasicBlockDefinition( MBB->setHasAddressTaken(); MBB->setIsEHPad(IsLandingPad); MBB->setIsEHFuncletEntry(IsEHFuncletEntry); - if (SectionID.hasValue()) { - MBB->setSectionID(SectionID.getValue()); + if (SectionType != MBBS_None) { + MBB->setSectionType(SectionType); MF.setBBSectionsType(BasicBlockSection::List); } return false; diff --git a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp index eb17434bc46338..5b6f71a2a01359 100644 --- a/llvm/lib/CodeGen/MIRParser/MIRParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIRParser.cpp @@ -442,8 +442,8 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF, MF.createBBLabels(); MF.setBBSectionsType(BasicBlockSection::Labels); } else if (MF.hasBBSections()) { + MF.setSectionRange(); MF.createBBLabels(); - MF.assignBeginEndSections(); } PFS.SM = &SM; diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index 55044802791571..0853181e742676 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -643,18 +643,24 @@ void MIPrinter::print(const MachineBasicBlock &MBB) { OS << "align " << MBB.getAlignment().value(); HasAttributes = true; } - if (MBB.getSectionID() != MBBSectionID(0)) { + if (MBB.getSectionType() != MBBS_None) { OS << (HasAttributes ? ", " : " ("); OS << "bbsections "; - switch (MBB.getSectionID().Type) { - case MBBSectionID::SectionType::Exception: + switch (MBB.getSectionType()) { + case MBBS_Entry: + OS << "Entry"; + break; + case MBBS_Exception: OS << "Exception"; break; - case MBBSectionID::SectionType::Cold: + case MBBS_Cold: OS << "Cold"; break; + case MBBS_Unique: + OS << "Unique"; + break; default: - OS << MBB.getSectionID().Number; + llvm_unreachable("No such section type"); } HasAttributes = true; } diff --git a/llvm/lib/CodeGen/MachineBasicBlock.cpp b/llvm/lib/CodeGen/MachineBasicBlock.cpp index 5f46086c9395e4..3fa60dd45aa2d1 100644 --- a/llvm/lib/CodeGen/MachineBasicBlock.cpp +++ b/llvm/lib/CodeGen/MachineBasicBlock.cpp @@ -62,9 +62,7 @@ MCSymbol *MachineBasicBlock::getSymbol() const { MCContext &Ctx = MF->getContext(); auto Prefix = Ctx.getAsmInfo()->getPrivateLabelPrefix(); - // We emit a non-temporary symbol for every basic block if we have BBLabels - // or -- with basic block sections -- when a basic block begins a section. - bool BasicBlockSymbols = isBeginSection() || MF->hasBBLabels(); + bool BasicBlockSymbols = MF->hasBBSections() || MF->hasBBLabels(); auto Delimiter = BasicBlockSymbols ? "." : "_"; assert(getNumber() >= 0 && "cannot get label for unreachable MBB"); @@ -550,6 +548,48 @@ void MachineBasicBlock::moveAfter(MachineBasicBlock *NewBefore) { getParent()->splice(++NewBefore->getIterator(), getIterator()); } +// Returns true if this basic block and the Other are in the same section. +bool MachineBasicBlock::sameSection(const MachineBasicBlock *Other) const { + if (this == Other) + return true; + + if (this->getSectionType() != Other->getSectionType()) + return false; + + // If either is in a unique section, return false. + if (this->getSectionType() == llvm::MachineBasicBlockSection::MBBS_Unique || + Other->getSectionType() == llvm::MachineBasicBlockSection::MBBS_Unique) + return false; + + return true; +} + +const MachineBasicBlock *MachineBasicBlock::getSectionEndMBB() const { + if (this->isEndSection()) + return this; + auto I = std::next(this->getIterator()); + const MachineFunction *MF = getParent(); + while (I != MF->end()) { + const MachineBasicBlock &MBB = *I; + if (MBB.isEndSection()) + return &MBB; + I = std::next(I); + } + llvm_unreachable("No End Basic Block for this section."); +} + +// Returns true if this block begins any section. +bool MachineBasicBlock::isBeginSection() const { + return (SectionType == MBBS_Entry || SectionType == MBBS_Unique || + getParent()->isSectionStartMBB(getNumber())); +} + +// Returns true if this block begins any section. +bool MachineBasicBlock::isEndSection() const { + return (SectionType == MBBS_Entry || SectionType == MBBS_Unique || + getParent()->isSectionEndMBB(getNumber())); +} + void MachineBasicBlock::updateTerminator() { const TargetInstrInfo *TII = getParent()->getSubtarget().getInstrInfo(); // A block with no successors has no concerns with fall-through edges. @@ -1532,7 +1572,3 @@ MachineBasicBlock::livein_iterator MachineBasicBlock::livein_begin() const { "Liveness information is accurate"); return LiveIns.begin(); } - -const MBBSectionID MBBSectionID::ColdSectionID(MBBSectionID::SectionType::Cold); -const MBBSectionID - MBBSectionID::ExceptionSectionID(MBBSectionID::SectionType::Exception); diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index 464f389c310175..1b89333b98f3e8 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -340,6 +340,32 @@ void MachineFunction::RenumberBlocks(MachineBasicBlock *MBB) { MBBNumbering.resize(BlockNo); } +/// This sets the section ranges of cold or exception section with basic block +/// sections. +void MachineFunction::setSectionRange() { + // Compute the Section Range of cold and exception basic blocks. Find the + // first and last block of each range. + auto SectionRange = + ([&](llvm::MachineBasicBlockSection S) -> std::pair { + auto MBBP = + std::find_if(begin(), end(), [&](MachineBasicBlock &MBB) -> bool { + return MBB.getSectionType() == S; + }); + if (MBBP == end()) + return std::make_pair(-1, -1); + + auto MBBQ = + std::find_if(rbegin(), rend(), [&](MachineBasicBlock &MBB) -> bool { + return MBB.getSectionType() == S; + }); + assert(MBBQ != rend() && "Section end not found!"); + return std::make_pair(MBBP->getNumber(), MBBQ->getNumber()); + }); + + ExceptionSectionRange = SectionRange(MBBS_Exception); + ColdSectionRange = SectionRange(llvm::MBBS_Cold); +} + /// This is used with -fbasicblock-sections or -fbasicblock-labels option. /// A unary encoding of basic block labels is done to keep ".strtab" sizes /// small. @@ -367,22 +393,6 @@ void MachineFunction::createBBLabels() { } } -/// This method iterates over the basic blocks and assigns their IsBeginSection -/// and IsEndSection fields. This must be called after MBB layout is finalized -/// and the SectionID's are assigned to MBBs. -void MachineFunction::assignBeginEndSections() { - front().setIsBeginSection(); - auto CurrentSectionID = front().getSectionID(); - for (auto MBBI = std::next(begin()), E = end(); MBBI != E; ++MBBI) { - if (MBBI->getSectionID() == CurrentSectionID) - continue; - MBBI->setIsBeginSection(); - std::prev(MBBI)->setIsEndSection(); - CurrentSectionID = MBBI->getSectionID(); - } - back().setIsEndSection(); -} - /// Allocate a new MachineInstr. Use this instead of `new MachineInstr'. MachineInstr *MachineFunction::CreateMachineInstr(const MCInstrDesc &MCID, const DebugLoc &DL, diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 17946696a26f47..b96cba456e7c34 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -769,31 +769,14 @@ MCSection *TargetLoweringObjectFileELF::getSectionForConstant( MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock( const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM) const { - assert(MBB.isBeginSection() && "Basic block does not start a section!"); SmallString<128> Name; Name = (static_cast(MBB.getParent()->getSection())) ->getSectionName(); - unsigned UniqueID = MCContext::GenericSectionID; - - switch (MBB.getSectionID().Type) { - // Append suffixes to represent special cold and exception sections. - case MBBSectionID::SectionType::Exception: - Name += ".eh"; - break; - case MBBSectionID::SectionType::Cold: - Name += ".unlikely"; - break; - // For regular sections, either use a unique name, or a unique ID for the - // section. - default: - if (TM.getUniqueBBSectionNames()) { - Name += "."; - Name += MBB.getSymbol()->getName(); - } else - UniqueID = NextUniqueID++; - break; + if (TM.getUniqueBBSectionNames()) { + Name += "."; + Name += MBB.getSymbol()->getName(); } - + unsigned UniqueID = NextUniqueID++; unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR; std::string GroupName = ""; if (F.hasComdat()) { @@ -805,6 +788,33 @@ MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock( nullptr); } +MCSection *TargetLoweringObjectFileELF::getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const { + SmallString<128> Name; + Name = (static_cast(MBB.getParent()->getSection())) + ->getSectionName(); + + // If unique section names is off, explicity add the function name to the + // section name to make sure named sections for functions are unique + // across the module. + if (!TM.getUniqueSectionNames()) { + Name += "."; + Name += MBB.getParent()->getName(); + } + + Name += Suffix; + + unsigned Flags = ELF::SHF_ALLOC | ELF::SHF_EXECINSTR; + std::string GroupName = ""; + if (F.hasComdat()) { + Flags |= ELF::SHF_GROUP; + GroupName = F.getComdat()->getName().str(); + } + return getContext().getELFSection(Name, ELF::SHT_PROGBITS, Flags, + 0 /* Entry Size */, GroupName); +} + static MCSectionELF *getStaticStructorSection(MCContext &Ctx, bool UseInitArray, bool IsCtor, unsigned Priority, const MCSymbol *KeySym) { diff --git a/llvm/lib/Target/TargetLoweringObjectFile.cpp b/llvm/lib/Target/TargetLoweringObjectFile.cpp index e7312e70cfb205..92caa6f940907c 100644 --- a/llvm/lib/Target/TargetLoweringObjectFile.cpp +++ b/llvm/lib/Target/TargetLoweringObjectFile.cpp @@ -311,6 +311,12 @@ MCSection *TargetLoweringObjectFile::getSectionForMachineBasicBlock( return nullptr; } +MCSection *TargetLoweringObjectFile::getNamedSectionForMachineBasicBlock( + const Function &F, const MachineBasicBlock &MBB, const TargetMachine &TM, + const char *Suffix) const { + return nullptr; +} + /// getTTypeGlobalReference - Return an MCExpr to use for a /// reference to the specified global variable from exception /// handling information. diff --git a/llvm/test/CodeGen/X86/basicblock-sections-cold.ll b/llvm/test/CodeGen/X86/basicblock-sections-cold.ll index 114ea8fe539d7f..5ddaf8f7cf7f3a 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-cold.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections-cold.ll @@ -37,5 +37,5 @@ declare i32 @_Z3foov() #1 ; LINUX-SECTIONS: .section .text._Z3bazb.unlikely,"ax",@progbits ; LINUX-SECTIONS: r.BB._Z3bazb: ; LINUX-SECTIONS-NOT: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique -; LINUX-SECTIONS: .LBB0_2: +; LINUX-SECTIONS: rr.BB._Z3bazb: ; LINUX-SECTIONS: .size _Z3bazb, .Lfunc_end{{[0-9]}}-_Z3bazb diff --git a/llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll b/llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll index c961f8d3273801..7038b689ab506d 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections-directjumps.ll @@ -30,9 +30,9 @@ declare i32 @_Z3foov() #1 ; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits ; LINUX-SECTIONS: _Z3bazb: ; LINUX-SECTIONS: jmp a.BB._Z3bazb -; LINUX-SECTIONS: .section .text._Z3bazb.a.BB._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: .section .text._Z3bazb.a.BB._Z3bazb,"ax",@progbits,unique,1 ; LINUX-SECTIONS: a.BB._Z3bazb: ; LINUX-SECTIONS: jmp aa.BB._Z3bazb -; LINUX-SECTIONS: .section .text._Z3bazb.aa.BB._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: .section .text._Z3bazb.aa.BB._Z3bazb,"ax",@progbits,unique,2 ; LINUX-SECTIONS: aa.BB._Z3bazb: ; LINUX-SECTIONS: jmp raa.BB._Z3bazb diff --git a/llvm/test/CodeGen/X86/basicblock-sections-eh.ll b/llvm/test/CodeGen/X86/basicblock-sections-eh.ll index 2d1fd88cbd4d2e..239ac4000fe6da 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-eh.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections-eh.ll @@ -80,5 +80,5 @@ declare void @__cxa_end_catch() ;LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits ;LINUX-SECTIONS: _Z3foob: -;LINUX-SECTIONS: .section .text._Z3foob.laara.BB._Z3foob,"ax",@progbits +;LINUX-SECTIONS: .section .text._Z3foob.eh,"ax",@progbits ;LINUX-SECTIONS: l{{[a|r]*}}.BB._Z3foob: diff --git a/llvm/test/CodeGen/X86/basicblock-sections-list.ll b/llvm/test/CodeGen/X86/basicblock-sections-list.ll index 55524ce6b454b7..93db874165cf3c 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-list.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections-list.ll @@ -59,18 +59,18 @@ define i32 @_Z3zipb(i1 zeroext %0) #0 { ; LINUX-SECTIONS: .section .text._Z3foob,"ax",@progbits ; LINUX-SECTIONS: _Z3foob: -; LINUX-SECTIONS: .section .text._Z3foob.a.BB._Z3foob,"ax",@progbits +; LINUX-SECTIONS: .section .text._Z3foob.a.BB._Z3foob,"ax",@progbits,unique,1 ; LINUX-SECTIONS: a.BB._Z3foob: -; LINUX-SECTIONS: .section .text._Z3foob.aa.BB._Z3foob,"ax",@progbits +; LINUX-SECTIONS: .section .text._Z3foob.aa.BB._Z3foob,"ax",@progbits,unique,2 ; LINUX-SECTIONS: aa.BB._Z3foob: -; LINUX-SECTIONS: .section .text._Z3foob.raa.BB._Z3foob,"ax",@progbits +; LINUX-SECTIONS: .section .text._Z3foob.raa.BB._Z3foob,"ax",@progbits,unique,3 ; LINUX-SECTIONS: raa.BB._Z3foob: ; LINUX-SECTIONS: .section .text._Z3zipb,"ax",@progbits ; LINUX-SECTIONS: _Z3zipb: -; LINUX-SECTIONS-NOT: .section .text._Z3zipb.a.BB._Z3zipb,"ax",@progbits +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.a.BB._Z3zipb,"ax",@progbits,unique,1 ; LINUX-SECTIONS-NOT: a.BB._Z3zipb: -; LINUX-SECTIONS-NOT: .section .text._Z3zipb.aa.BB._Z3zipb,"ax",@progbits +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.aa.BB._Z3zipb,"ax",@progbits,unique,2 ; LINUX-SECTIONS-NOT: aa.BB._Z3zipb: -; LINUX-SECTIONS-NOT: .section .text._Z3zipb.raa.BB._Z3zipb,"ax",@progbits +; LINUX-SECTIONS-NOT: .section .text._Z3zipb.raa.BB._Z3zipb,"ax",@progbits,unique,3 ; LINUX-SECTIONS-NOT: raa.BB._Z3zipb: diff --git a/llvm/test/CodeGen/X86/basicblock-sections-listbb.ll b/llvm/test/CodeGen/X86/basicblock-sections-listbb.ll index a93ca9fdc2ffbe..9bc8cacb4aa927 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-listbb.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections-listbb.ll @@ -31,9 +31,8 @@ declare i32 @_Z3foov() #1 ; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits ; LINUX-SECTIONS: _Z3bazb: ; Check that the basic block with id 1 doesn't get a section. -; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits -; LINUX-SECTIONS: # %bb.1: -; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits +; LINUX-SECTIONS-NOT: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique +; LINUX-SECTIONS: r.BB._Z3bazb: +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique ; LINUX-SECTIONS: rr.BB._Z3bazb: -; LINUX-SECTIONS: .Ltmp0: -; LINUX-SECTIONS-NEXT: .size rr.BB._Z3bazb, .Ltmp0-rr.BB._Z3bazb +; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb diff --git a/llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir b/llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir index 0c80b32e143c02..81db8b03272880 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir +++ b/llvm/test/CodeGen/X86/basicblock-sections-mir-parse.mir @@ -87,7 +87,7 @@ callSites: [] constants: [] machineFunctionInfo: {} body: | - bb.0 (%ir-block.1, align 4, bbsections 0): + bb.0 (%ir-block.1, bbsections Entry): successors: %bb.2(0x40000000), %bb.1(0x40000000) liveins: $edi @@ -102,19 +102,19 @@ body: | JCC_1 %bb.2, 4, implicit killed $eflags JMP_1 %bb.1 - bb.1 (%ir-block.7, bbsections 1): + bb.1 (%ir-block.7, bbsections Unique): successors: %bb.3(0x80000000) MOV32mi $rbp, 1, $noreg, -8, $noreg, 1 :: (store 4 into %ir.2) JMP_1 %bb.3 - bb.2 (%ir-block.8, bbsections 2): + bb.2 (%ir-block.8, bbsections Unique): successors: %bb.3(0x80000000) MOV32mi $rbp, 1, $noreg, -8, $noreg, 0 :: (store 4 into %ir.2) JMP_1 %bb.3 - bb.3 (%ir-block.9, bbsections 3): + bb.3 (%ir-block.9, bbsections Unique): renamable $eax = MOV32rm $rbp, 1, $noreg, -8, $noreg :: (load 4 from %ir.2) $rbp = frame-destroy POP64r implicit-def $rsp, implicit $rsp CFI_INSTRUCTION def_cfa $rsp, 8 diff --git a/llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll b/llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll index bd66f44ecc6db9..93ac4d1af942e0 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections-mir-print.ll @@ -1,7 +1,6 @@ ; Stop after bbsections-prepare and check MIR output for section type. ; RUN: echo '!_Z3foob' > %t ; RUN: echo '!!1' >> %t -; RUN: echo '!!2' >> %t ; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basicblock-sections=%t -stop-after=bbsections-prepare | FileCheck %s -check-prefix=CHECK @_ZTIb = external constant i8* @@ -27,7 +26,7 @@ define dso_local i32 @_Z3foob(i1 zeroext %0) { ret i32 %10 } -; CHECK: bb.0 (%ir-block.1, bbsections Cold): -; CHECK: bb.3 (%ir-block.9, bbsections Cold): -; CHECK: bb.1 (%ir-block.7) -; CHECK: bb.2 (%ir-block.8, bbsections 1): +; CHECK: bbsections Entry +; CHECK: bbsections Cold +; CHECK: bbsections Cold +; CHECK: bbsections Unique diff --git a/llvm/test/CodeGen/X86/basicblock-sections.ll b/llvm/test/CodeGen/X86/basicblock-sections.ll index 85b7e1fc9ac5c1..3ba4d4ac928264 100644 --- a/llvm/test/CodeGen/X86/basicblock-sections.ll +++ b/llvm/test/CodeGen/X86/basicblock-sections.ll @@ -28,11 +28,9 @@ declare i32 @_Z3foov() #1 ; LINUX-SECTIONS: .section .text._Z3bazb,"ax",@progbits ; LINUX-SECTIONS: _Z3bazb: -; LINUX-SECTIONS: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: .section .text._Z3bazb.r.BB._Z3bazb,"ax",@progbits,unique,1 ; LINUX-SECTIONS: r.BB._Z3bazb: -; LINUX-SECTIONS: .Ltmp0: -; LINUX-SECTIONS-NEXT: .size r.BB._Z3bazb, .Ltmp0-r.BB._Z3bazb -; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits +; LINUX-SECTIONS: .size r.BB._Z3bazb, .Ltmp0-r.BB._Z3bazb +; LINUX-SECTIONS: .section .text._Z3bazb.rr.BB._Z3bazb,"ax",@progbits,unique,2 ; LINUX-SECTIONS: rr.BB._Z3bazb: -; LINUX-SECTIONS: .Ltmp1: -; LINUX-SECTIONS-NEXT: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb +; LINUX-SECTIONS: .size rr.BB._Z3bazb, .Ltmp1-rr.BB._Z3bazb