Skip to content

Commit

Permalink
[BOLT] Support passing fragments to code emission
Browse files Browse the repository at this point in the history 
This changes code emission such that it can emit specific function
fragments instead of scanning all basic blocks of a function and just
emitting those that are hot or cold.

To implement this, `FunctionLayout` explicitly distinguishes the "main"
fragment (i.e. the one that contains the entry block and is associated
with the original symbol) from "split" fragments. Additionally,
`BinaryFunction` receives support for multiple cold symbols - one for
each split fragment.

Reviewed By: rafauler

Differential Revision: https://reviews.llvm.org/D130052
  • Loading branch information
@FPar
FPar committed Aug 19, 2022
1 parent 89d7db9 commit 275e075
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 112 deletions.
3 changes: 2 additions & 1 deletion bolt/include/bolt/Core/BinaryEmitter.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ class MCStreamer;
namespace bolt {
class BinaryContext;
class BinaryFunction;
class FunctionFragment;

/// Emit all code and data from the BinaryContext \p BC into the \p Streamer.
///
Expand All @@ -34,7 +35,7 @@ void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
/// Emit \p BF function code. The caller is responsible for emitting function
/// symbol(s) and setting the section to emit the code to.
void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
bool EmitColdPart, bool EmitCodeOnly = false);
const FunctionFragment &FF, bool EmitCodeOnly);

} // namespace bolt
} // namespace llvm
Expand Down
42 changes: 26 additions & 16 deletions bolt/include/bolt/Core/BinaryFunction.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -568,7 +568,7 @@ class BinaryFunction {
};
SmallVector<BasicBlockOffset, 0> BasicBlockOffsets;

MCSymbol *ColdSymbol{nullptr};
SmallVector<MCSymbol *, 0> ColdSymbols;

/// Symbol at the end of the function.
mutable MCSymbol *FunctionEndLabel{nullptr};
Expand Down Expand Up @@ -1081,7 +1081,23 @@ class BinaryFunction {

/// Return MC symbol associated with the function.
/// All references to the function should use this symbol.
MCSymbol *getSymbol() { return Symbols[0]; }
MCSymbol *getSymbol(const FragmentNum Fragment = FragmentNum::hot()) {
if (Fragment == FragmentNum::hot())
return Symbols[0];

size_t ColdSymbolIndex = Fragment.get() - 1;
if (ColdSymbolIndex >= ColdSymbols.size())
ColdSymbols.resize(ColdSymbolIndex + 1);

MCSymbol *&ColdSymbol = ColdSymbols[ColdSymbolIndex];
if (ColdSymbol == nullptr) {
SmallString<10> Appendix = formatv(".cold.{0}", ColdSymbolIndex);
ColdSymbol = BC.Ctx->getOrCreateSymbol(
NameResolver::append(Symbols[0]->getName(), Appendix));
}

return ColdSymbol;
}

/// Return MC symbol associated with the function (const version).
/// All references to the function should use this symbol.
Expand Down Expand Up @@ -1135,16 +1151,6 @@ class BinaryFunction {
/// Return true of all callbacks returned true, false otherwise.
bool forEachEntryPoint(EntryPointCallbackTy Callback) const;

MCSymbol *getColdSymbol() {
if (ColdSymbol)
return ColdSymbol;

ColdSymbol = BC.Ctx->getOrCreateSymbol(
NameResolver::append(getSymbol()->getName(), ".cold.0"));

return ColdSymbol;
}

/// Return MC symbol associated with the end of the function.
MCSymbol *getFunctionEndLabel() const {
assert(BC.Ctx && "cannot be called with empty context");
Expand Down Expand Up @@ -1323,8 +1329,11 @@ class BinaryFunction {
}

/// Return cold code section name for the function.
StringRef getColdCodeSectionName() const {
return StringRef(ColdCodeSectionName);
std::string getColdCodeSectionName(const FragmentNum Fragment) const {
std::string Result = ColdCodeSectionName;
if (Fragment != FragmentNum::cold())
Result.append(".").append(std::to_string(Fragment.get() - 1));
return Result;
}

/// Assign a section name for the cold part of the function.
Expand All @@ -1333,8 +1342,9 @@ class BinaryFunction {
}

/// Get output code section for cold code of this function.
ErrorOr<BinarySection &> getColdCodeSection() const {
return BC.getUniqueSectionByName(getColdCodeSectionName());
ErrorOr<BinarySection &>
getColdCodeSection(const FragmentNum Fragment) const {
return BC.getUniqueSectionByName(getColdCodeSectionName(Fragment));
}

/// Return true iif the function will halt execution on entry.
Expand Down
16 changes: 16 additions & 0 deletions bolt/include/bolt/Core/FunctionLayout.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@ class FunctionFragment {
: Num(Num), Layout(Layout) {}

public:
FragmentNum getFragmentNum() const { return Num; }
bool isMainFragment() const { return Num.get() == 0; }
bool isSplitFragment() const { return Num.get() > 0; }

unsigned size() const;
bool empty() const;
const_iterator begin() const;
Expand Down Expand Up @@ -153,6 +157,18 @@ class FunctionLayout {
/// Return the fragment identified by Num.
FunctionFragment getFragment(FragmentNum Num) const;

/// Get the fragment that contains all entry blocks and other blocks that
/// cannot be split.
FunctionFragment getMainFragment() const {
return getFragment(FragmentNum::hot());
}

/// Get the fragment that contains all entry blocks and other blocks that
/// cannot be split.
iterator_range<const_iterator> getSplitFragments() const {
return {++fragment_begin(), fragment_end()};
}

/// Find the fragment that contains BB.
FunctionFragment findFragment(const BinaryBasicBlock *BB) const;

Expand Down
45 changes: 26 additions & 19 deletions bolt/lib/Core/BinaryContext.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
#include <algorithm>
#include <functional>
#include <iterator>
#include <numeric>
#include <unordered_set>

using namespace llvm;
Expand Down Expand Up @@ -2189,27 +2190,31 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
// Create symbols in the LocalCtx so that they get destroyed with it.
MCSymbol *StartLabel = LocalCtx->createTempSymbol();
MCSymbol *EndLabel = LocalCtx->createTempSymbol();
MCSymbol *ColdStartLabel = LocalCtx->createTempSymbol();
MCSymbol *ColdEndLabel = LocalCtx->createTempSymbol();

Streamer->switchSection(Section);
Streamer->emitLabel(StartLabel);
emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/false,
emitFunctionBody(*Streamer, BF, BF.getLayout().getMainFragment(),
/*EmitCodeOnly=*/true);
Streamer->emitLabel(EndLabel);

if (BF.isSplit()) {
MCSectionELF *ColdSection =
LocalCtx->getELFSection(BF.getColdCodeSectionName(), ELF::SHT_PROGBITS,
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
ColdSection->setHasInstructions(true);

Streamer->switchSection(ColdSection);
Streamer->emitLabel(ColdStartLabel);
emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/true,
/*EmitCodeOnly=*/true);
Streamer->emitLabel(ColdEndLabel);
// To avoid calling MCObjectStreamer::flushPendingLabels() which is private
using LabelRange = std::pair<const MCSymbol *, const MCSymbol *>;
SmallVector<LabelRange> SplitLabels;
for (const FunctionFragment FF : BF.getLayout().getSplitFragments()) {
MCSymbol *const SplitStartLabel = LocalCtx->createTempSymbol();
MCSymbol *const SplitEndLabel = LocalCtx->createTempSymbol();
SplitLabels.emplace_back(SplitStartLabel, SplitEndLabel);

MCSectionELF *const SplitSection = LocalCtx->getELFSection(
BF.getColdCodeSectionName(FF.getFragmentNum()), ELF::SHT_PROGBITS,
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
SplitSection->setHasInstructions(true);
Streamer->switchSection(SplitSection);

Streamer->emitLabel(SplitStartLabel);
emitFunctionBody(*Streamer, BF, FF, /*EmitCodeOnly=*/true);
Streamer->emitLabel(SplitEndLabel);
// To avoid calling MCObjectStreamer::flushPendingLabels() which is
// private
Streamer->emitBytes(StringRef(""));
Streamer->switchSection(Section);
}
Expand All @@ -2225,10 +2230,12 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {

const uint64_t HotSize =
Layout.getSymbolOffset(*EndLabel) - Layout.getSymbolOffset(*StartLabel);
const uint64_t ColdSize = BF.isSplit()
? Layout.getSymbolOffset(*ColdEndLabel) -
Layout.getSymbolOffset(*ColdStartLabel)
: 0ULL;
const uint64_t ColdSize =
std::accumulate(SplitLabels.begin(), SplitLabels.end(), 0ULL,
[&](const uint64_t Accu, const LabelRange &Labels) {
return Accu + Layout.getSymbolOffset(*Labels.second) -
Layout.getSymbolOffset(*Labels.first);
});

// Clean-up the effect of the code emission.
for (const MCSymbol &Symbol : Assembler.symbols()) {
Expand Down
76 changes: 46 additions & 30 deletions bolt/lib/Core/BinaryEmitter.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,15 +129,15 @@ class BinaryEmitter {

/// Emit function code. The caller is responsible for emitting function
/// symbol(s) and setting the section to emit the code to.
void emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
void emitFunctionBody(BinaryFunction &BF, const FunctionFragment &FF,
bool EmitCodeOnly = false);

private:
/// Emit function code.
void emitFunctions();

/// Emit a single function.
bool emitFunction(BinaryFunction &BF, bool EmitColdPart);
bool emitFunction(BinaryFunction &BF, const FunctionFragment &FF);

/// Helper for emitFunctionBody to write data inside a function
/// (used for AArch64)
Expand Down Expand Up @@ -234,13 +234,24 @@ void BinaryEmitter::emitFunctions() {
!Function->hasValidProfile())
Streamer.setAllowAutoPadding(false);

Emitted |= emitFunction(*Function, /*EmitColdPart=*/false);
const FunctionLayout &Layout = Function->getLayout();
Emitted |= emitFunction(*Function, Layout.getMainFragment());

if (Function->isSplit()) {
if (opts::X86AlignBranchBoundaryHotOnly)
Streamer.setAllowAutoPadding(false);
Emitted |= emitFunction(*Function, /*EmitColdPart=*/true);

assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
"Only simple functions can have fragments");
for (const FunctionFragment FF : Layout.getSplitFragments()) {
// Skip empty fragments so no symbols and sections for empty fragments
// are generated
if (FF.empty() && !Function->hasConstantIsland())
continue;
Emitted |= emitFunction(*Function, FF);
}
}

Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);

if (Emitted)
Expand Down Expand Up @@ -268,16 +279,18 @@ void BinaryEmitter::emitFunctions() {
}
}

bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
bool BinaryEmitter::emitFunction(BinaryFunction &Function,
const FunctionFragment &FF) {
if (Function.size() == 0 && !Function.hasIslandsInfo())
return false;

if (Function.getState() == BinaryFunction::State::Empty)
return false;

MCSection *Section =
BC.getCodeSection(EmitColdPart ? Function.getColdCodeSectionName()
: Function.getCodeSectionName());
MCSection *Section = BC.getCodeSection(
FF.isSplitFragment()
? Function.getColdCodeSectionName(FF.getFragmentNum())
: Function.getCodeSectionName());
Streamer.switchSection(Section);
Section->setHasInstructions(true);
BC.Ctx->addGenDwarfSection(Section);
Expand All @@ -290,8 +303,9 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
Section->setAlignment(Align(opts::AlignFunctions));

Streamer.emitCodeAlignment(BinaryFunction::MinAlign, &*BC.STI);
uint16_t MaxAlignBytes = EmitColdPart ? Function.getMaxColdAlignmentBytes()
: Function.getMaxAlignmentBytes();
uint16_t MaxAlignBytes = FF.isSplitFragment()
? Function.getMaxColdAlignmentBytes()
: Function.getMaxAlignmentBytes();
if (MaxAlignBytes > 0)
Streamer.emitCodeAlignment(Function.getAlignment(), &*BC.STI,
MaxAlignBytes);
Expand All @@ -302,29 +316,29 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
MCContext &Context = Streamer.getContext();
const MCAsmInfo *MAI = Context.getAsmInfo();

MCSymbol *StartSymbol = nullptr;
MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());

// Emit all symbols associated with the main function entry.
if (!EmitColdPart) {
StartSymbol = Function.getSymbol();
if (FF.isMainFragment()) {
for (MCSymbol *Symbol : Function.getSymbols()) {
Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
Streamer.emitLabel(Symbol);
}
} else {
StartSymbol = Function.getColdSymbol();
Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
Streamer.emitLabel(StartSymbol);
}

// Emit CFI start
if (Function.hasCFI()) {
assert(Function.getLayout().isHotColdSplit() &&
"Exceptions supported only with hot/cold splitting.");
Streamer.emitCFIStartProc(/*IsSimple=*/false);
if (Function.getPersonalityFunction() != nullptr)
Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
Function.getPersonalityEncoding());
MCSymbol *LSDASymbol =
EmitColdPart ? Function.getColdLSDASymbol() : Function.getLSDASymbol();
MCSymbol *LSDASymbol = FF.isSplitFragment() ? Function.getColdLSDASymbol()
: Function.getLSDASymbol();
if (LSDASymbol)
Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
else
Expand Down Expand Up @@ -353,7 +367,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
}

// Emit code.
emitFunctionBody(Function, EmitColdPart, /*EmitCodeOnly=*/false);
emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);

// Emit padding if requested.
if (size_t Padding = opts::padFunction(Function)) {
Expand All @@ -369,8 +383,9 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
if (Function.hasCFI())
Streamer.emitCFIEndProc();

MCSymbol *EndSymbol = EmitColdPart ? Function.getFunctionColdEndLabel()
: Function.getFunctionEndLabel();
MCSymbol *EndSymbol = FF.isSplitFragment()
? Function.getFunctionColdEndLabel()
: Function.getFunctionEndLabel();
Streamer.emitLabel(EndSymbol);

if (MAI->hasDotTypeDotSizeDirective()) {
Expand All @@ -384,21 +399,22 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
emitLineInfoEnd(Function, EndSymbol);

// Exception handling info for the function.
emitLSDA(Function, EmitColdPart);
emitLSDA(Function, FF.isSplitFragment());

if (!EmitColdPart && opts::JumpTables > JTS_NONE)
if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
emitJumpTables(Function);

return true;
}

void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
void BinaryEmitter::emitFunctionBody(BinaryFunction &BF,
const FunctionFragment &FF,
bool EmitCodeOnly) {
if (!EmitCodeOnly && EmitColdPart && BF.hasConstantIsland())
if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
assert(FF.getFragmentNum() == FragmentNum::cold() &&
"Constant island support only with hot/cold split");
BF.duplicateConstantIslands();

const FunctionFragment FF = BF.getLayout().getFragment(
EmitColdPart ? FragmentNum::cold() : FragmentNum::hot());
}

if (!FF.empty() && FF.front()->isLandingPad()) {
assert(!FF.front()->isEntryPoint() &&
Expand Down Expand Up @@ -488,7 +504,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
}

if (!EmitCodeOnly)
emitConstantIslands(BF, EmitColdPart);
emitConstantIslands(BF, FF.isSplitFragment());
}

void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
Expand Down Expand Up @@ -904,7 +920,7 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {

// Corresponding FDE start.
const MCSymbol *StartSymbol =
EmitColdPart ? BF.getColdSymbol() : BF.getSymbol();
EmitColdPart ? BF.getSymbol(FragmentNum::cold()) : BF.getSymbol();

// Emit the LSDA header.

Expand Down Expand Up @@ -1148,9 +1164,9 @@ void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
}

void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
bool EmitColdPart, bool EmitCodeOnly) {
const FunctionFragment &FF, bool EmitCodeOnly) {
BinaryEmitter(Streamer, BF.getBinaryContext())
.emitFunctionBody(BF, EmitColdPart, EmitCodeOnly);
.emitFunctionBody(BF, FF, EmitCodeOnly);
}

} // namespace bolt
Expand Down
Loading

0 comments on commit 275e075

Please sign in to comment.