Skip to content

Commit 275e075

Browse files
author
Fabian Parzefall
committed
[BOLT] Support passing fragments to code emission
This changes code emission such that it can emit specific function fragments instead of scanning all basic blocks of a function and just emitting those that are hot or cold. To implement this, `FunctionLayout` explicitly distinguishes the "main" fragment (i.e. the one that contains the entry block and is associated with the original symbol) from "split" fragments. Additionally, `BinaryFunction` receives support for multiple cold symbols - one for each split fragment. Reviewed By: rafauler Differential Revision: https://reviews.llvm.org/D130052
1 parent 89d7db9 commit 275e075

File tree

7 files changed

+176
-112
lines changed

7 files changed

+176
-112
lines changed

bolt/include/bolt/Core/BinaryEmitter.h

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ class MCStreamer;
2222
namespace bolt {
2323
class BinaryContext;
2424
class BinaryFunction;
25+
class FunctionFragment;
2526

2627
/// Emit all code and data from the BinaryContext \p BC into the \p Streamer.
2728
///
@@ -34,7 +35,7 @@ void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
3435
/// Emit \p BF function code. The caller is responsible for emitting function
3536
/// symbol(s) and setting the section to emit the code to.
3637
void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
37-
bool EmitColdPart, bool EmitCodeOnly = false);
38+
const FunctionFragment &FF, bool EmitCodeOnly);
3839

3940
} // namespace bolt
4041
} // namespace llvm

bolt/include/bolt/Core/BinaryFunction.h

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -568,7 +568,7 @@ class BinaryFunction {
568568
};
569569
SmallVector<BasicBlockOffset, 0> BasicBlockOffsets;
570570

571-
MCSymbol *ColdSymbol{nullptr};
571+
SmallVector<MCSymbol *, 0> ColdSymbols;
572572

573573
/// Symbol at the end of the function.
574574
mutable MCSymbol *FunctionEndLabel{nullptr};
@@ -1081,7 +1081,23 @@ class BinaryFunction {
10811081

10821082
/// Return MC symbol associated with the function.
10831083
/// All references to the function should use this symbol.
1084-
MCSymbol *getSymbol() { return Symbols[0]; }
1084+
MCSymbol *getSymbol(const FragmentNum Fragment = FragmentNum::hot()) {
1085+
if (Fragment == FragmentNum::hot())
1086+
return Symbols[0];
1087+
1088+
size_t ColdSymbolIndex = Fragment.get() - 1;
1089+
if (ColdSymbolIndex >= ColdSymbols.size())
1090+
ColdSymbols.resize(ColdSymbolIndex + 1);
1091+
1092+
MCSymbol *&ColdSymbol = ColdSymbols[ColdSymbolIndex];
1093+
if (ColdSymbol == nullptr) {
1094+
SmallString<10> Appendix = formatv(".cold.{0}", ColdSymbolIndex);
1095+
ColdSymbol = BC.Ctx->getOrCreateSymbol(
1096+
NameResolver::append(Symbols[0]->getName(), Appendix));
1097+
}
1098+
1099+
return ColdSymbol;
1100+
}
10851101

10861102
/// Return MC symbol associated with the function (const version).
10871103
/// All references to the function should use this symbol.
@@ -1135,16 +1151,6 @@ class BinaryFunction {
11351151
/// Return true of all callbacks returned true, false otherwise.
11361152
bool forEachEntryPoint(EntryPointCallbackTy Callback) const;
11371153

1138-
MCSymbol *getColdSymbol() {
1139-
if (ColdSymbol)
1140-
return ColdSymbol;
1141-
1142-
ColdSymbol = BC.Ctx->getOrCreateSymbol(
1143-
NameResolver::append(getSymbol()->getName(), ".cold.0"));
1144-
1145-
return ColdSymbol;
1146-
}
1147-
11481154
/// Return MC symbol associated with the end of the function.
11491155
MCSymbol *getFunctionEndLabel() const {
11501156
assert(BC.Ctx && "cannot be called with empty context");
@@ -1323,8 +1329,11 @@ class BinaryFunction {
13231329
}
13241330

13251331
/// Return cold code section name for the function.
1326-
StringRef getColdCodeSectionName() const {
1327-
return StringRef(ColdCodeSectionName);
1332+
std::string getColdCodeSectionName(const FragmentNum Fragment) const {
1333+
std::string Result = ColdCodeSectionName;
1334+
if (Fragment != FragmentNum::cold())
1335+
Result.append(".").append(std::to_string(Fragment.get() - 1));
1336+
return Result;
13281337
}
13291338

13301339
/// Assign a section name for the cold part of the function.
@@ -1333,8 +1342,9 @@ class BinaryFunction {
13331342
}
13341343

13351344
/// Get output code section for cold code of this function.
1336-
ErrorOr<BinarySection &> getColdCodeSection() const {
1337-
return BC.getUniqueSectionByName(getColdCodeSectionName());
1345+
ErrorOr<BinarySection &>
1346+
getColdCodeSection(const FragmentNum Fragment) const {
1347+
return BC.getUniqueSectionByName(getColdCodeSectionName(Fragment));
13381348
}
13391349

13401350
/// Return true iif the function will halt execution on entry.

bolt/include/bolt/Core/FunctionLayout.h

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ class FunctionFragment {
6666
: Num(Num), Layout(Layout) {}
6767

6868
public:
69+
FragmentNum getFragmentNum() const { return Num; }
70+
bool isMainFragment() const { return Num.get() == 0; }
71+
bool isSplitFragment() const { return Num.get() > 0; }
72+
6973
unsigned size() const;
7074
bool empty() const;
7175
const_iterator begin() const;
@@ -153,6 +157,18 @@ class FunctionLayout {
153157
/// Return the fragment identified by Num.
154158
FunctionFragment getFragment(FragmentNum Num) const;
155159

160+
/// Get the fragment that contains all entry blocks and other blocks that
161+
/// cannot be split.
162+
FunctionFragment getMainFragment() const {
163+
return getFragment(FragmentNum::hot());
164+
}
165+
166+
/// Get the fragment that contains all entry blocks and other blocks that
167+
/// cannot be split.
168+
iterator_range<const_iterator> getSplitFragments() const {
169+
return {++fragment_begin(), fragment_end()};
170+
}
171+
156172
/// Find the fragment that contains BB.
157173
FunctionFragment findFragment(const BinaryBasicBlock *BB) const;
158174

bolt/lib/Core/BinaryContext.cpp

Lines changed: 26 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
#include <algorithm>
3939
#include <functional>
4040
#include <iterator>
41+
#include <numeric>
4142
#include <unordered_set>
4243

4344
using namespace llvm;
@@ -2189,27 +2190,31 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
21892190
// Create symbols in the LocalCtx so that they get destroyed with it.
21902191
MCSymbol *StartLabel = LocalCtx->createTempSymbol();
21912192
MCSymbol *EndLabel = LocalCtx->createTempSymbol();
2192-
MCSymbol *ColdStartLabel = LocalCtx->createTempSymbol();
2193-
MCSymbol *ColdEndLabel = LocalCtx->createTempSymbol();
21942193

21952194
Streamer->switchSection(Section);
21962195
Streamer->emitLabel(StartLabel);
2197-
emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/false,
2196+
emitFunctionBody(*Streamer, BF, BF.getLayout().getMainFragment(),
21982197
/*EmitCodeOnly=*/true);
21992198
Streamer->emitLabel(EndLabel);
22002199

2201-
if (BF.isSplit()) {
2202-
MCSectionELF *ColdSection =
2203-
LocalCtx->getELFSection(BF.getColdCodeSectionName(), ELF::SHT_PROGBITS,
2204-
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
2205-
ColdSection->setHasInstructions(true);
2206-
2207-
Streamer->switchSection(ColdSection);
2208-
Streamer->emitLabel(ColdStartLabel);
2209-
emitFunctionBody(*Streamer, BF, /*EmitColdPart=*/true,
2210-
/*EmitCodeOnly=*/true);
2211-
Streamer->emitLabel(ColdEndLabel);
2212-
// To avoid calling MCObjectStreamer::flushPendingLabels() which is private
2200+
using LabelRange = std::pair<const MCSymbol *, const MCSymbol *>;
2201+
SmallVector<LabelRange> SplitLabels;
2202+
for (const FunctionFragment FF : BF.getLayout().getSplitFragments()) {
2203+
MCSymbol *const SplitStartLabel = LocalCtx->createTempSymbol();
2204+
MCSymbol *const SplitEndLabel = LocalCtx->createTempSymbol();
2205+
SplitLabels.emplace_back(SplitStartLabel, SplitEndLabel);
2206+
2207+
MCSectionELF *const SplitSection = LocalCtx->getELFSection(
2208+
BF.getColdCodeSectionName(FF.getFragmentNum()), ELF::SHT_PROGBITS,
2209+
ELF::SHF_EXECINSTR | ELF::SHF_ALLOC);
2210+
SplitSection->setHasInstructions(true);
2211+
Streamer->switchSection(SplitSection);
2212+
2213+
Streamer->emitLabel(SplitStartLabel);
2214+
emitFunctionBody(*Streamer, BF, FF, /*EmitCodeOnly=*/true);
2215+
Streamer->emitLabel(SplitEndLabel);
2216+
// To avoid calling MCObjectStreamer::flushPendingLabels() which is
2217+
// private
22132218
Streamer->emitBytes(StringRef(""));
22142219
Streamer->switchSection(Section);
22152220
}
@@ -2225,10 +2230,12 @@ BinaryContext::calculateEmittedSize(BinaryFunction &BF, bool FixBranches) {
22252230

22262231
const uint64_t HotSize =
22272232
Layout.getSymbolOffset(*EndLabel) - Layout.getSymbolOffset(*StartLabel);
2228-
const uint64_t ColdSize = BF.isSplit()
2229-
? Layout.getSymbolOffset(*ColdEndLabel) -
2230-
Layout.getSymbolOffset(*ColdStartLabel)
2231-
: 0ULL;
2233+
const uint64_t ColdSize =
2234+
std::accumulate(SplitLabels.begin(), SplitLabels.end(), 0ULL,
2235+
[&](const uint64_t Accu, const LabelRange &Labels) {
2236+
return Accu + Layout.getSymbolOffset(*Labels.second) -
2237+
Layout.getSymbolOffset(*Labels.first);
2238+
});
22322239

22332240
// Clean-up the effect of the code emission.
22342241
for (const MCSymbol &Symbol : Assembler.symbols()) {

bolt/lib/Core/BinaryEmitter.cpp

Lines changed: 46 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -129,15 +129,15 @@ class BinaryEmitter {
129129

130130
/// Emit function code. The caller is responsible for emitting function
131131
/// symbol(s) and setting the section to emit the code to.
132-
void emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
132+
void emitFunctionBody(BinaryFunction &BF, const FunctionFragment &FF,
133133
bool EmitCodeOnly = false);
134134

135135
private:
136136
/// Emit function code.
137137
void emitFunctions();
138138

139139
/// Emit a single function.
140-
bool emitFunction(BinaryFunction &BF, bool EmitColdPart);
140+
bool emitFunction(BinaryFunction &BF, const FunctionFragment &FF);
141141

142142
/// Helper for emitFunctionBody to write data inside a function
143143
/// (used for AArch64)
@@ -234,13 +234,24 @@ void BinaryEmitter::emitFunctions() {
234234
!Function->hasValidProfile())
235235
Streamer.setAllowAutoPadding(false);
236236

237-
Emitted |= emitFunction(*Function, /*EmitColdPart=*/false);
237+
const FunctionLayout &Layout = Function->getLayout();
238+
Emitted |= emitFunction(*Function, Layout.getMainFragment());
238239

239240
if (Function->isSplit()) {
240241
if (opts::X86AlignBranchBoundaryHotOnly)
241242
Streamer.setAllowAutoPadding(false);
242-
Emitted |= emitFunction(*Function, /*EmitColdPart=*/true);
243+
244+
assert((Layout.fragment_size() == 1 || Function->isSimple()) &&
245+
"Only simple functions can have fragments");
246+
for (const FunctionFragment FF : Layout.getSplitFragments()) {
247+
// Skip empty fragments so no symbols and sections for empty fragments
248+
// are generated
249+
if (FF.empty() && !Function->hasConstantIsland())
250+
continue;
251+
Emitted |= emitFunction(*Function, FF);
252+
}
243253
}
254+
244255
Streamer.setAllowAutoPadding(OriginalAllowAutoPadding);
245256

246257
if (Emitted)
@@ -268,16 +279,18 @@ void BinaryEmitter::emitFunctions() {
268279
}
269280
}
270281

271-
bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
282+
bool BinaryEmitter::emitFunction(BinaryFunction &Function,
283+
const FunctionFragment &FF) {
272284
if (Function.size() == 0 && !Function.hasIslandsInfo())
273285
return false;
274286

275287
if (Function.getState() == BinaryFunction::State::Empty)
276288
return false;
277289

278-
MCSection *Section =
279-
BC.getCodeSection(EmitColdPart ? Function.getColdCodeSectionName()
280-
: Function.getCodeSectionName());
290+
MCSection *Section = BC.getCodeSection(
291+
FF.isSplitFragment()
292+
? Function.getColdCodeSectionName(FF.getFragmentNum())
293+
: Function.getCodeSectionName());
281294
Streamer.switchSection(Section);
282295
Section->setHasInstructions(true);
283296
BC.Ctx->addGenDwarfSection(Section);
@@ -290,8 +303,9 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
290303
Section->setAlignment(Align(opts::AlignFunctions));
291304

292305
Streamer.emitCodeAlignment(BinaryFunction::MinAlign, &*BC.STI);
293-
uint16_t MaxAlignBytes = EmitColdPart ? Function.getMaxColdAlignmentBytes()
294-
: Function.getMaxAlignmentBytes();
306+
uint16_t MaxAlignBytes = FF.isSplitFragment()
307+
? Function.getMaxColdAlignmentBytes()
308+
: Function.getMaxAlignmentBytes();
295309
if (MaxAlignBytes > 0)
296310
Streamer.emitCodeAlignment(Function.getAlignment(), &*BC.STI,
297311
MaxAlignBytes);
@@ -302,29 +316,29 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
302316
MCContext &Context = Streamer.getContext();
303317
const MCAsmInfo *MAI = Context.getAsmInfo();
304318

305-
MCSymbol *StartSymbol = nullptr;
319+
MCSymbol *const StartSymbol = Function.getSymbol(FF.getFragmentNum());
306320

307321
// Emit all symbols associated with the main function entry.
308-
if (!EmitColdPart) {
309-
StartSymbol = Function.getSymbol();
322+
if (FF.isMainFragment()) {
310323
for (MCSymbol *Symbol : Function.getSymbols()) {
311324
Streamer.emitSymbolAttribute(Symbol, MCSA_ELF_TypeFunction);
312325
Streamer.emitLabel(Symbol);
313326
}
314327
} else {
315-
StartSymbol = Function.getColdSymbol();
316328
Streamer.emitSymbolAttribute(StartSymbol, MCSA_ELF_TypeFunction);
317329
Streamer.emitLabel(StartSymbol);
318330
}
319331

320332
// Emit CFI start
321333
if (Function.hasCFI()) {
334+
assert(Function.getLayout().isHotColdSplit() &&
335+
"Exceptions supported only with hot/cold splitting.");
322336
Streamer.emitCFIStartProc(/*IsSimple=*/false);
323337
if (Function.getPersonalityFunction() != nullptr)
324338
Streamer.emitCFIPersonality(Function.getPersonalityFunction(),
325339
Function.getPersonalityEncoding());
326-
MCSymbol *LSDASymbol =
327-
EmitColdPart ? Function.getColdLSDASymbol() : Function.getLSDASymbol();
340+
MCSymbol *LSDASymbol = FF.isSplitFragment() ? Function.getColdLSDASymbol()
341+
: Function.getLSDASymbol();
328342
if (LSDASymbol)
329343
Streamer.emitCFILsda(LSDASymbol, BC.LSDAEncoding);
330344
else
@@ -353,7 +367,7 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
353367
}
354368

355369
// Emit code.
356-
emitFunctionBody(Function, EmitColdPart, /*EmitCodeOnly=*/false);
370+
emitFunctionBody(Function, FF, /*EmitCodeOnly=*/false);
357371

358372
// Emit padding if requested.
359373
if (size_t Padding = opts::padFunction(Function)) {
@@ -369,8 +383,9 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
369383
if (Function.hasCFI())
370384
Streamer.emitCFIEndProc();
371385

372-
MCSymbol *EndSymbol = EmitColdPart ? Function.getFunctionColdEndLabel()
373-
: Function.getFunctionEndLabel();
386+
MCSymbol *EndSymbol = FF.isSplitFragment()
387+
? Function.getFunctionColdEndLabel()
388+
: Function.getFunctionEndLabel();
374389
Streamer.emitLabel(EndSymbol);
375390

376391
if (MAI->hasDotTypeDotSizeDirective()) {
@@ -384,21 +399,22 @@ bool BinaryEmitter::emitFunction(BinaryFunction &Function, bool EmitColdPart) {
384399
emitLineInfoEnd(Function, EndSymbol);
385400

386401
// Exception handling info for the function.
387-
emitLSDA(Function, EmitColdPart);
402+
emitLSDA(Function, FF.isSplitFragment());
388403

389-
if (!EmitColdPart && opts::JumpTables > JTS_NONE)
404+
if (FF.isMainFragment() && opts::JumpTables > JTS_NONE)
390405
emitJumpTables(Function);
391406

392407
return true;
393408
}
394409

395-
void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
410+
void BinaryEmitter::emitFunctionBody(BinaryFunction &BF,
411+
const FunctionFragment &FF,
396412
bool EmitCodeOnly) {
397-
if (!EmitCodeOnly && EmitColdPart && BF.hasConstantIsland())
413+
if (!EmitCodeOnly && FF.isSplitFragment() && BF.hasConstantIsland()) {
414+
assert(FF.getFragmentNum() == FragmentNum::cold() &&
415+
"Constant island support only with hot/cold split");
398416
BF.duplicateConstantIslands();
399-
400-
const FunctionFragment FF = BF.getLayout().getFragment(
401-
EmitColdPart ? FragmentNum::cold() : FragmentNum::hot());
417+
}
402418

403419
if (!FF.empty() && FF.front()->isLandingPad()) {
404420
assert(!FF.front()->isEntryPoint() &&
@@ -488,7 +504,7 @@ void BinaryEmitter::emitFunctionBody(BinaryFunction &BF, bool EmitColdPart,
488504
}
489505

490506
if (!EmitCodeOnly)
491-
emitConstantIslands(BF, EmitColdPart);
507+
emitConstantIslands(BF, FF.isSplitFragment());
492508
}
493509

494510
void BinaryEmitter::emitConstantIslands(BinaryFunction &BF, bool EmitColdPart,
@@ -904,7 +920,7 @@ void BinaryEmitter::emitLSDA(BinaryFunction &BF, bool EmitColdPart) {
904920

905921
// Corresponding FDE start.
906922
const MCSymbol *StartSymbol =
907-
EmitColdPart ? BF.getColdSymbol() : BF.getSymbol();
923+
EmitColdPart ? BF.getSymbol(FragmentNum::cold()) : BF.getSymbol();
908924

909925
// Emit the LSDA header.
910926

@@ -1148,9 +1164,9 @@ void emitBinaryContext(MCStreamer &Streamer, BinaryContext &BC,
11481164
}
11491165

11501166
void emitFunctionBody(MCStreamer &Streamer, BinaryFunction &BF,
1151-
bool EmitColdPart, bool EmitCodeOnly) {
1167+
const FunctionFragment &FF, bool EmitCodeOnly) {
11521168
BinaryEmitter(Streamer, BF.getBinaryContext())
1153-
.emitFunctionBody(BF, EmitColdPart, EmitCodeOnly);
1169+
.emitFunctionBody(BF, FF, EmitCodeOnly);
11541170
}
11551171

11561172
} // namespace bolt

0 commit comments

Comments
 (0)