25 changes: 13 additions & 12 deletions bolt/lib/Passes/PatchEntries.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ llvm::cl::opt<bool>
namespace llvm {
namespace bolt {

void PatchEntries::runOnFunctions(BinaryContext &BC) {
Error PatchEntries::runOnFunctions(BinaryContext &BC) {
if (!opts::ForcePatch) {
// Mark the binary for patching if we did not create external references
// for original code in any of functions we are not going to emit.
Expand All @@ -42,11 +42,11 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {
});

if (!NeedsPatching)
return;
return Error::success();
}

if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: patching entries in original code\n";
BC.outs() << "BOLT-INFO: patching entries in original code\n";

// Calculate the size of the patch.
static size_t PatchSize = 0;
Expand Down Expand Up @@ -78,8 +78,8 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {
const MCSymbol *Symbol) {
if (Offset < NextValidByte) {
if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: unable to patch entry point in " << Function
<< " at offset 0x" << Twine::utohexstr(Offset) << '\n';
BC.outs() << "BOLT-INFO: unable to patch entry point in " << Function
<< " at offset 0x" << Twine::utohexstr(Offset) << '\n';
return false;
}

Expand All @@ -89,8 +89,8 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {
NextValidByte = Offset + PatchSize;
if (NextValidByte > Function.getMaxSize()) {
if (opts::Verbosity >= 1)
outs() << "BOLT-INFO: function " << Function
<< " too small to patch its entry point\n";
BC.outs() << "BOLT-INFO: function " << Function
<< " too small to patch its entry point\n";
return false;
}

Expand All @@ -101,18 +101,18 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {
// We can't change output layout for AArch64 due to LongJmp pass
if (BC.isAArch64()) {
if (opts::ForcePatch) {
errs() << "BOLT-ERROR: unable to patch entries in " << Function
<< "\n";
exit(1);
BC.errs() << "BOLT-ERROR: unable to patch entries in " << Function
<< "\n";
return createFatalBOLTError("");
}

continue;
}

// If the original function entries cannot be patched, then we cannot
// safely emit new function body.
errs() << "BOLT-WARNING: failed to patch entries in " << Function
<< ". The function will not be optimized.\n";
BC.errs() << "BOLT-WARNING: failed to patch entries in " << Function
<< ". The function will not be optimized.\n";
Function.setIgnored();
continue;
}
Expand All @@ -138,6 +138,7 @@ void PatchEntries::runOnFunctions(BinaryContext &BC) {

Function.setIsPatched(true);
}
return Error::success();
}

} // end namespace bolt
Expand Down
10 changes: 5 additions & 5 deletions bolt/lib/Passes/RegAnalysis.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -232,11 +232,11 @@ BitVector RegAnalysis::getFunctionClobberList(const BinaryFunction *Func) {
}

void RegAnalysis::printStats() {
outs() << "BOLT-INFO REG ANALYSIS: Number of functions conservatively "
"treated as clobbering all registers: "
<< NumFunctionsAllClobber
<< format(" (%.1lf%% dyn cov)\n",
(100.0 * CountFunctionsAllClobber / CountDenominator));
BC.outs() << "BOLT-INFO REG ANALYSIS: Number of functions conservatively "
"treated as clobbering all registers: "
<< NumFunctionsAllClobber
<< format(" (%.1lf%% dyn cov)\n",
(100.0 * CountFunctionsAllClobber / CountDenominator));
}

} // namespace bolt
Expand Down
24 changes: 13 additions & 11 deletions bolt/lib/Passes/RegReAssign.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -452,7 +452,7 @@ void RegReAssign::setupConservativePass(
});
}

void RegReAssign::runOnFunctions(BinaryContext &BC) {
Error RegReAssign::runOnFunctions(BinaryContext &BC) {
RegScore = std::vector<int64_t>(BC.MRI->getNumRegs(), 0);
RankedRegs = std::vector<size_t>(BC.MRI->getNumRegs(), 0);

Expand Down Expand Up @@ -480,18 +480,20 @@ void RegReAssign::runOnFunctions(BinaryContext &BC) {
}

if (FuncsChanged.empty()) {
outs() << "BOLT-INFO: Reg Reassignment Pass: no changes were made.\n";
return;
BC.outs() << "BOLT-INFO: Reg Reassignment Pass: no changes were made.\n";
return Error::success();
}
if (opts::UpdateDebugSections)
outs() << "BOLT-WARNING: You used -reg-reassign and -update-debug-sections."
<< " Some registers were changed but associated AT_LOCATION for "
<< "impacted variables were NOT updated! This operation is "
<< "currently unsupported by BOLT.\n";
outs() << "BOLT-INFO: Reg Reassignment Pass Stats:\n";
outs() << "\t " << FuncsChanged.size() << " functions affected.\n";
outs() << "\t " << StaticBytesSaved << " static bytes saved.\n";
outs() << "\t " << DynBytesSaved << " dynamic bytes saved.\n";
BC.outs()
<< "BOLT-WARNING: You used -reg-reassign and -update-debug-sections."
<< " Some registers were changed but associated AT_LOCATION for "
<< "impacted variables were NOT updated! This operation is "
<< "currently unsupported by BOLT.\n";
BC.outs() << "BOLT-INFO: Reg Reassignment Pass Stats:\n";
BC.outs() << "\t " << FuncsChanged.size() << " functions affected.\n";
BC.outs() << "\t " << StaticBytesSaved << " static bytes saved.\n";
BC.outs() << "\t " << DynBytesSaved << " dynamic bytes saved.\n";
return Error::success();
}

} // namespace bolt
Expand Down
65 changes: 34 additions & 31 deletions bolt/lib/Passes/ReorderData.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ bool filterSymbol(const BinaryData *BD) {

using DataOrder = ReorderData::DataOrder;

void ReorderData::printOrder(const BinarySection &Section,
void ReorderData::printOrder(BinaryContext &BC, const BinarySection &Section,
DataOrder::const_iterator Begin,
DataOrder::const_iterator End) const {
uint64_t TotalSize = 0;
Expand All @@ -142,19 +142,20 @@ void ReorderData::printOrder(const BinarySection &Section,
const BinaryData *BD = Begin->first;

if (!PrintHeader) {
outs() << "BOLT-INFO: Hot global symbols for " << Section.getName()
<< ":\n";
BC.outs() << "BOLT-INFO: Hot global symbols for " << Section.getName()
<< ":\n";
PrintHeader = true;
}

outs() << "BOLT-INFO: " << *BD << ", moveable=" << BD->isMoveable()
<< format(", weight=%.5f\n", double(Begin->second) / BD->getSize());
BC.outs() << "BOLT-INFO: " << *BD << ", moveable=" << BD->isMoveable()
<< format(", weight=%.5f\n",
double(Begin->second) / BD->getSize());

TotalSize += BD->getSize();
++Begin;
}
if (TotalSize)
outs() << "BOLT-INFO: Total hot symbol size = " << TotalSize << "\n";
BC.outs() << "BOLT-INFO: Total hot symbol size = " << TotalSize << "\n";
}

DataOrder ReorderData::baseOrder(BinaryContext &BC,
Expand Down Expand Up @@ -208,19 +209,19 @@ void ReorderData::assignMemData(BinaryContext &BC) {
}

if (!Counts.empty()) {
outs() << "BOLT-INFO: Memory stats breakdown:\n";
BC.outs() << "BOLT-INFO: Memory stats breakdown:\n";
for (const auto &KV : Counts) {
StringRef Section = KV.first;
const uint64_t Count = KV.second;
outs() << "BOLT-INFO: " << Section << " = " << Count
<< format(" (%.1f%%)\n", 100.0 * Count / TotalCount);
BC.outs() << "BOLT-INFO: " << Section << " = " << Count
<< format(" (%.1f%%)\n", 100.0 * Count / TotalCount);
if (JumpTableCounts.count(Section) != 0) {
const uint64_t JTCount = JumpTableCounts[Section];
outs() << "BOLT-INFO: jump tables = " << JTCount
<< format(" (%.1f%%)\n", 100.0 * JTCount / Count);
BC.outs() << "BOLT-INFO: jump tables = " << JTCount
<< format(" (%.1f%%)\n", 100.0 * JTCount / Count);
}
}
outs() << "BOLT-INFO: Total memory events: " << TotalCount << "\n";
BC.outs() << "BOLT-INFO: Total memory events: " << TotalCount << "\n";
}
}

Expand Down Expand Up @@ -395,9 +396,9 @@ void ReorderData::setSectionOrder(BinaryContext &BC,

OutputSection.reorderContents(NewOrder, opts::ReorderInplace);

outs() << "BOLT-INFO: reorder-data: " << Count << "/" << TotalCount
<< format(" (%.1f%%)", 100.0 * Count / TotalCount) << " events, "
<< Offset << " hot bytes\n";
BC.outs() << "BOLT-INFO: reorder-data: " << Count << "/" << TotalCount
<< format(" (%.1f%%)", 100.0 * Count / TotalCount) << " events, "
<< Offset << " hot bytes\n";
}

bool ReorderData::markUnmoveableSymbols(BinaryContext &BC,
Expand Down Expand Up @@ -435,17 +436,17 @@ bool ReorderData::markUnmoveableSymbols(BinaryContext &BC,
return FoundUnmoveable;
}

void ReorderData::runOnFunctions(BinaryContext &BC) {
Error ReorderData::runOnFunctions(BinaryContext &BC) {
static const char *DefaultSections[] = {".rodata", ".data", ".bss", nullptr};

if (!BC.HasRelocations || opts::ReorderData.empty())
return;
return Error::success();

// For now
if (opts::JumpTables > JTS_BASIC) {
outs() << "BOLT-WARNING: jump table support must be basic for "
<< "data reordering to work.\n";
return;
BC.outs() << "BOLT-WARNING: jump table support must be basic for "
<< "data reordering to work.\n";
return Error::success();
}

assignMemData(BC);
Expand All @@ -463,14 +464,14 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {

ErrorOr<BinarySection &> Section = BC.getUniqueSectionByName(SectionName);
if (!Section) {
outs() << "BOLT-WARNING: Section " << SectionName
<< " not found, skipping.\n";
BC.outs() << "BOLT-WARNING: Section " << SectionName
<< " not found, skipping.\n";
continue;
}

if (!isSupported(*Section)) {
outs() << "BOLT-ERROR: Section " << SectionName << " not supported.\n";
exit(1);
BC.errs() << "BOLT-ERROR: Section " << SectionName << " not supported.\n";
return createFatalBOLTError("");
}

Sections.push_back(&*Section);
Expand All @@ -483,23 +484,23 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
unsigned SplitPointIdx;

if (opts::ReorderAlgorithm == opts::ReorderAlgo::REORDER_COUNT) {
outs() << "BOLT-INFO: reorder-sections: ordering data by count\n";
BC.outs() << "BOLT-INFO: reorder-sections: ordering data by count\n";
std::tie(Order, SplitPointIdx) = sortedByCount(BC, *Section);
} else {
outs() << "BOLT-INFO: reorder-sections: ordering data by funcs\n";
BC.outs() << "BOLT-INFO: reorder-sections: ordering data by funcs\n";
std::tie(Order, SplitPointIdx) =
sortedByFunc(BC, *Section, BC.getBinaryFunctions());
}
auto SplitPoint = Order.begin() + SplitPointIdx;

if (opts::PrintReorderedData)
printOrder(*Section, Order.begin(), SplitPoint);
printOrder(BC, *Section, Order.begin(), SplitPoint);

if (!opts::ReorderInplace || FoundUnmoveable) {
if (opts::ReorderInplace && FoundUnmoveable)
outs() << "BOLT-INFO: Found unmoveable symbols in "
<< Section->getName() << " falling back to splitting "
<< "instead of in-place reordering.\n";
BC.outs() << "BOLT-INFO: Found unmoveable symbols in "
<< Section->getName() << " falling back to splitting "
<< "instead of in-place reordering.\n";

// Rename sections.
BinarySection &Hot =
Expand All @@ -519,10 +520,12 @@ void ReorderData::runOnFunctions(BinaryContext &BC) {
}
}
} else {
outs() << "BOLT-WARNING: Inplace section reordering not supported yet.\n";
BC.outs()
<< "BOLT-WARNING: Inplace section reordering not supported yet.\n";
setSectionOrder(BC, *Section, Order.begin(), Order.end());
}
}
return Error::success();
}

} // namespace bolt
Expand Down
143 changes: 76 additions & 67 deletions bolt/lib/Passes/ReorderFunctions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,8 @@ using NodeId = CallGraph::NodeId;
using Arc = CallGraph::Arc;
using Node = CallGraph::Node;

void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
void ReorderFunctions::reorder(BinaryContext &BC,
std::vector<Cluster> &&Clusters,
std::map<uint64_t, BinaryFunction> &BFs) {
std::vector<uint64_t> FuncAddr(Cg.numNodes()); // Just for computing stats
uint64_t TotalSize = 0;
Expand All @@ -139,10 +140,11 @@ void ReorderFunctions::reorder(std::vector<Cluster> &&Clusters,
if (opts::ReorderFunctions == RT_NONE)
return;

printStats(Clusters, FuncAddr);
printStats(BC, Clusters, FuncAddr);
}

void ReorderFunctions::printStats(const std::vector<Cluster> &Clusters,
void ReorderFunctions::printStats(BinaryContext &BC,
const std::vector<Cluster> &Clusters,
const std::vector<uint64_t> &FuncAddr) {
if (opts::Verbosity == 0) {
#ifndef NDEBUG
Expand All @@ -167,11 +169,11 @@ void ReorderFunctions::printStats(const std::vector<Cluster> &Clusters,
double TotalCalls4KB = 0;
double TotalCalls2MB = 0;
if (PrintDetailed)
outs() << "BOLT-INFO: Function reordering page layout\n"
<< "BOLT-INFO: ============== page 0 ==============\n";
BC.outs() << "BOLT-INFO: Function reordering page layout\n"
<< "BOLT-INFO: ============== page 0 ==============\n";
for (const Cluster &Cluster : Clusters) {
if (PrintDetailed)
outs() << format(
BC.outs() << format(
"BOLT-INFO: -------- density = %.3lf (%u / %u) --------\n",
Cluster.density(), Cluster.samples(), Cluster.size());

Expand All @@ -180,8 +182,8 @@ void ReorderFunctions::printStats(const std::vector<Cluster> &Clusters,
Hotfuncs++;

if (PrintDetailed)
outs() << "BOLT-INFO: hot func " << *Cg.nodeIdToFunc(FuncId) << " ("
<< Cg.size(FuncId) << ")\n";
BC.outs() << "BOLT-INFO: hot func " << *Cg.nodeIdToFunc(FuncId)
<< " (" << Cg.size(FuncId) << ")\n";

uint64_t Dist = 0;
uint64_t Calls = 0;
Expand All @@ -193,12 +195,13 @@ void ReorderFunctions::printStats(const std::vector<Cluster> &Clusters,
(FuncAddr[FuncId] + Arc.avgCallOffset()));
const double W = Arc.weight();
if (D < 64 && PrintDetailed && opts::Verbosity > 2)
outs() << "BOLT-INFO: short (" << D << "B) call:\n"
<< "BOLT-INFO: Src: " << *Cg.nodeIdToFunc(FuncId) << "\n"
<< "BOLT-INFO: Dst: " << *Cg.nodeIdToFunc(Dst) << "\n"
<< "BOLT-INFO: Weight = " << W << "\n"
<< "BOLT-INFO: AvgOffset = " << Arc.avgCallOffset()
<< "\n";
BC.outs() << "BOLT-INFO: short (" << D << "B) call:\n"
<< "BOLT-INFO: Src: " << *Cg.nodeIdToFunc(FuncId)
<< "\n"
<< "BOLT-INFO: Dst: " << *Cg.nodeIdToFunc(Dst) << "\n"
<< "BOLT-INFO: Weight = " << W << "\n"
<< "BOLT-INFO: AvgOffset = " << Arc.avgCallOffset()
<< "\n";
Calls += W;
if (D < 64)
TotalCalls64B += W;
Expand All @@ -208,63 +211,64 @@ void ReorderFunctions::printStats(const std::vector<Cluster> &Clusters,
TotalCalls2MB += W;
Dist += Arc.weight() * D;
if (PrintDetailed)
outs() << format("BOLT-INFO: arc: %u [@%lu+%.1lf] -> %u [@%lu]: "
"weight = %.0lf, callDist = %f\n",
Arc.src(), FuncAddr[Arc.src()],
Arc.avgCallOffset(), Arc.dst(),
FuncAddr[Arc.dst()], Arc.weight(), D);
BC.outs() << format("BOLT-INFO: arc: %u [@%lu+%.1lf] -> %u [@%lu]: "
"weight = %.0lf, callDist = %f\n",
Arc.src(), FuncAddr[Arc.src()],
Arc.avgCallOffset(), Arc.dst(),
FuncAddr[Arc.dst()], Arc.weight(), D);
}
TotalCalls += Calls;
TotalDistance += Dist;
TotalSize += Cg.size(FuncId);

if (PrintDetailed) {
outs() << format("BOLT-INFO: start = %6u : avgCallDist = %lu : ",
TotalSize, Calls ? Dist / Calls : 0)
<< Cg.nodeIdToFunc(FuncId)->getPrintName() << '\n';
BC.outs() << format("BOLT-INFO: start = %6u : avgCallDist = %lu : ",
TotalSize, Calls ? Dist / Calls : 0)
<< Cg.nodeIdToFunc(FuncId)->getPrintName() << '\n';
const uint64_t NewPage = TotalSize / HugePageSize;
if (NewPage != CurPage) {
CurPage = NewPage;
outs() << format(
BC.outs() << format(
"BOLT-INFO: ============== page %u ==============\n", CurPage);
}
}
}
}
}
outs() << "BOLT-INFO: Function reordering stats\n"
<< format("BOLT-INFO: Number of hot functions: %u\n"
"BOLT-INFO: Number of clusters: %lu\n",
Hotfuncs, Clusters.size())
<< format("BOLT-INFO: Final average call distance = %.1lf "
"(%.0lf / %.0lf)\n",
TotalCalls ? TotalDistance / TotalCalls : 0, TotalDistance,
TotalCalls)
<< format("BOLT-INFO: Total Calls = %.0lf\n", TotalCalls);
BC.outs() << "BOLT-INFO: Function reordering stats\n"
<< format("BOLT-INFO: Number of hot functions: %u\n"
"BOLT-INFO: Number of clusters: %lu\n",
Hotfuncs, Clusters.size())
<< format("BOLT-INFO: Final average call distance = %.1lf "
"(%.0lf / %.0lf)\n",
TotalCalls ? TotalDistance / TotalCalls : 0,
TotalDistance, TotalCalls)
<< format("BOLT-INFO: Total Calls = %.0lf\n", TotalCalls);
if (TotalCalls)
outs() << format("BOLT-INFO: Total Calls within 64B = %.0lf (%.2lf%%)\n",
TotalCalls64B, 100 * TotalCalls64B / TotalCalls)
<< format("BOLT-INFO: Total Calls within 4KB = %.0lf (%.2lf%%)\n",
TotalCalls4KB, 100 * TotalCalls4KB / TotalCalls)
<< format("BOLT-INFO: Total Calls within 2MB = %.0lf (%.2lf%%)\n",
TotalCalls2MB, 100 * TotalCalls2MB / TotalCalls);
BC.outs()
<< format("BOLT-INFO: Total Calls within 64B = %.0lf (%.2lf%%)\n",
TotalCalls64B, 100 * TotalCalls64B / TotalCalls)
<< format("BOLT-INFO: Total Calls within 4KB = %.0lf (%.2lf%%)\n",
TotalCalls4KB, 100 * TotalCalls4KB / TotalCalls)
<< format("BOLT-INFO: Total Calls within 2MB = %.0lf (%.2lf%%)\n",
TotalCalls2MB, 100 * TotalCalls2MB / TotalCalls);
}

std::vector<std::string> ReorderFunctions::readFunctionOrderFile() {
std::vector<std::string> FunctionNames;
Error ReorderFunctions::readFunctionOrderFile(
std::vector<std::string> &FunctionNames) {
std::ifstream FuncsFile(opts::FunctionOrderFile, std::ios::in);
if (!FuncsFile) {
errs() << "Ordered functions file \"" << opts::FunctionOrderFile
<< "\" can't be opened.\n";
exit(1);
}
if (!FuncsFile)
return createFatalBOLTError(Twine("Ordered functions file \"") +
Twine(opts::FunctionOrderFile) +
Twine("\" can't be opened."));

std::string FuncName;
while (std::getline(FuncsFile, FuncName))
FunctionNames.push_back(FuncName);
return FunctionNames;
return Error::success();
}

void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
Error ReorderFunctions::runOnFunctions(BinaryContext &BC) {
auto &BFs = BC.getBinaryFunctions();
if (opts::ReorderFunctions != RT_NONE &&
opts::ReorderFunctions != RT_EXEC_COUNT &&
Expand Down Expand Up @@ -373,7 +377,11 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {

uint32_t Index = 0;
uint32_t InvalidEntries = 0;
for (const std::string &Function : readFunctionOrderFile()) {
std::vector<std::string> FunctionNames;
if (Error E = readFunctionOrderFile(FunctionNames))
return Error(std::move(E));

for (const std::string &Function : FunctionNames) {
std::vector<uint64_t> FuncAddrs;

BinaryData *BD = BC.getBinaryDataByName(Function);
Expand All @@ -399,8 +407,8 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {

if (FuncAddrs.empty()) {
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: Reorder functions: can't find function "
<< "for " << Function << "\n";
BC.errs() << "BOLT-WARNING: Reorder functions: can't find function "
<< "for " << Function << "\n";
++InvalidEntries;
continue;
}
Expand All @@ -412,28 +420,28 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
BinaryFunction *BF = BC.getFunctionForSymbol(FuncBD->getSymbol());
if (!BF) {
if (opts::Verbosity >= 1)
errs() << "BOLT-WARNING: Reorder functions: can't find function "
<< "for " << Function << "\n";
BC.errs() << "BOLT-WARNING: Reorder functions: can't find function "
<< "for " << Function << "\n";
++InvalidEntries;
break;
}
if (!BF->hasValidIndex())
BF->setIndex(Index++);
else if (opts::Verbosity > 0)
errs() << "BOLT-WARNING: Duplicate reorder entry for " << Function
<< "\n";
BC.errs() << "BOLT-WARNING: Duplicate reorder entry for " << Function
<< "\n";
}
}
if (InvalidEntries)
errs() << "BOLT-WARNING: Reorder functions: can't find functions for "
<< InvalidEntries << " entries in -function-order list\n";
BC.errs() << "BOLT-WARNING: Reorder functions: can't find functions for "
<< InvalidEntries << " entries in -function-order list\n";
} break;

default:
llvm_unreachable("unexpected layout type");
}

reorder(std::move(Clusters), BFs);
reorder(BC, std::move(Clusters), BFs);

BC.HasFinalizedFunctionOrder = true;

Expand All @@ -442,9 +450,9 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
FuncsFile = std::make_unique<std::ofstream>(opts::GenerateFunctionOrderFile,
std::ios::out);
if (!FuncsFile) {
errs() << "BOLT-ERROR: ordered functions file "
<< opts::GenerateFunctionOrderFile << " cannot be opened\n";
exit(1);
BC.errs() << "BOLT-ERROR: ordered functions file "
<< opts::GenerateFunctionOrderFile << " cannot be opened\n";
return createFatalBOLTError("");
}
}

Expand All @@ -453,9 +461,9 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {
LinkSectionsFile =
std::make_unique<std::ofstream>(opts::LinkSectionsFile, std::ios::out);
if (!LinkSectionsFile) {
errs() << "BOLT-ERROR: link sections file " << opts::LinkSectionsFile
<< " cannot be opened\n";
exit(1);
BC.errs() << "BOLT-ERROR: link sections file " << opts::LinkSectionsFile
<< " cannot be opened\n";
return createFatalBOLTError("");
}
}

Expand Down Expand Up @@ -505,16 +513,17 @@ void ReorderFunctions::runOnFunctions(BinaryContext &BC) {

if (FuncsFile) {
FuncsFile->close();
outs() << "BOLT-INFO: dumped function order to "
<< opts::GenerateFunctionOrderFile << '\n';
BC.outs() << "BOLT-INFO: dumped function order to "
<< opts::GenerateFunctionOrderFile << '\n';
}

if (LinkSectionsFile) {
LinkSectionsFile->close();
outs() << "BOLT-INFO: dumped linker section order to "
<< opts::LinkSectionsFile << '\n';
BC.outs() << "BOLT-INFO: dumped linker section order to "
<< opts::LinkSectionsFile << '\n';
}
}
return Error::success();
}

} // namespace bolt
Expand Down
13 changes: 7 additions & 6 deletions bolt/lib/Passes/RetpolineInsertion.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -271,9 +271,9 @@ IndirectBranchInfo::IndirectBranchInfo(MCInst &Inst, MCPlusBuilder &MIB) {
}
}

void RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
Error RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
if (!opts::InsertRetpolines)
return;
return Error::success();

assert(BC.isX86() &&
"retpoline insertion not supported for target architecture");
Expand Down Expand Up @@ -327,10 +327,11 @@ void RetpolineInsertion::runOnFunctions(BinaryContext &BC) {
}
}
}
outs() << "BOLT-INFO: The number of created retpoline functions is : "
<< CreatedRetpolines.size()
<< "\nBOLT-INFO: The number of retpolined branches is : "
<< RetpolinedBranches << "\n";
BC.outs() << "BOLT-INFO: The number of created retpoline functions is : "
<< CreatedRetpolines.size()
<< "\nBOLT-INFO: The number of retpolined branches is : "
<< RetpolinedBranches << "\n";
return Error::success();
}

} // namespace bolt
Expand Down
105 changes: 61 additions & 44 deletions bolt/lib/Passes/ShrinkWrapping.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1646,25 +1646,25 @@ void ShrinkWrapping::rebuildCFIForSP() {
++I;
}

MCInst ShrinkWrapping::createStackAccess(int SPVal, int FPVal,
const FrameIndexEntry &FIE,
bool CreatePushOrPop) {
Expected<MCInst> ShrinkWrapping::createStackAccess(int SPVal, int FPVal,
const FrameIndexEntry &FIE,
bool CreatePushOrPop) {
MCInst NewInst;
if (SPVal != StackPointerTracking::SUPERPOSITION &&
SPVal != StackPointerTracking::EMPTY) {
if (FIE.IsLoad) {
if (!BC.MIB->createRestoreFromStack(NewInst, BC.MIB->getStackPointer(),
FIE.StackOffset - SPVal, FIE.RegOrImm,
FIE.Size)) {
errs() << "createRestoreFromStack: not supported on this platform\n";
abort();
return createFatalBOLTError(
"createRestoreFromStack: not supported on this platform\n");
}
} else {
if (!BC.MIB->createSaveToStack(NewInst, BC.MIB->getStackPointer(),
FIE.StackOffset - SPVal, FIE.RegOrImm,
FIE.Size)) {
errs() << "createSaveToStack: not supported on this platform\n";
abort();
return createFatalBOLTError(
"createSaveToStack: not supported on this platform\n");
}
}
if (CreatePushOrPop)
Expand All @@ -1678,15 +1678,15 @@ MCInst ShrinkWrapping::createStackAccess(int SPVal, int FPVal,
if (!BC.MIB->createRestoreFromStack(NewInst, BC.MIB->getFramePointer(),
FIE.StackOffset - FPVal, FIE.RegOrImm,
FIE.Size)) {
errs() << "createRestoreFromStack: not supported on this platform\n";
abort();
return createFatalBOLTError(
"createRestoreFromStack: not supported on this platform\n");
}
} else {
if (!BC.MIB->createSaveToStack(NewInst, BC.MIB->getFramePointer(),
FIE.StackOffset - FPVal, FIE.RegOrImm,
FIE.Size)) {
errs() << "createSaveToStack: not supported on this platform\n";
abort();
return createFatalBOLTError(
"createSaveToStack: not supported on this platform\n");
}
}
return NewInst;
Expand Down Expand Up @@ -1743,10 +1743,11 @@ BBIterTy ShrinkWrapping::insertCFIsForPushOrPop(BinaryBasicBlock &BB,
return Pos;
}

BBIterTy ShrinkWrapping::processInsertion(BBIterTy InsertionPoint,
BinaryBasicBlock *CurBB,
const WorklistItem &Item,
int64_t SPVal, int64_t FPVal) {
Expected<BBIterTy> ShrinkWrapping::processInsertion(BBIterTy InsertionPoint,
BinaryBasicBlock *CurBB,
const WorklistItem &Item,
int64_t SPVal,
int64_t FPVal) {
// Trigger CFI reconstruction for this CSR if necessary - writing to
// PushOffsetByReg/PopOffsetByReg *will* trigger CFI update
if ((Item.FIEToInsert.IsStore &&
Expand All @@ -1772,9 +1773,12 @@ BBIterTy ShrinkWrapping::processInsertion(BBIterTy InsertionPoint,
<< " Is push = " << (Item.Action == WorklistItem::InsertPushOrPop)
<< "\n";
});
MCInst NewInst =
Expected<MCInst> NewInstOrErr =
createStackAccess(SPVal, FPVal, Item.FIEToInsert,
Item.Action == WorklistItem::InsertPushOrPop);
if (auto E = NewInstOrErr.takeError())
return Error(std::move(E));
MCInst &NewInst = *NewInstOrErr;
if (InsertionPoint != CurBB->end()) {
LLVM_DEBUG({
dbgs() << "Adding before Inst: ";
Expand All @@ -1791,7 +1795,7 @@ BBIterTy ShrinkWrapping::processInsertion(BBIterTy InsertionPoint,
return CurBB->end();
}

BBIterTy ShrinkWrapping::processInsertionsList(
Expected<BBIterTy> ShrinkWrapping::processInsertionsList(
BBIterTy InsertionPoint, BinaryBasicBlock *CurBB,
std::vector<WorklistItem> &TodoList, int64_t SPVal, int64_t FPVal) {
bool HasInsertions = llvm::any_of(TodoList, [&](WorklistItem &Item) {
Expand Down Expand Up @@ -1840,8 +1844,11 @@ BBIterTy ShrinkWrapping::processInsertionsList(
Item.Action == WorklistItem::ChangeToAdjustment)
continue;

InsertionPoint =
auto InsertionPointOrErr =
processInsertion(InsertionPoint, CurBB, Item, SPVal, FPVal);
if (auto E = InsertionPointOrErr.takeError())
return Error(std::move(E));
InsertionPoint = *InsertionPointOrErr;
if (Item.Action == WorklistItem::InsertPushOrPop &&
Item.FIEToInsert.IsStore)
SPVal -= Item.FIEToInsert.Size;
Expand All @@ -1852,7 +1859,7 @@ BBIterTy ShrinkWrapping::processInsertionsList(
return InsertionPoint;
}

bool ShrinkWrapping::processInsertions() {
Expected<bool> ShrinkWrapping::processInsertions() {
PredictiveStackPointerTracking PSPT(BF, Todo, Info, AllocatorId);
PSPT.run();

Expand All @@ -1875,14 +1882,20 @@ bool ShrinkWrapping::processInsertions() {
auto Iter = I;
std::pair<int, int> SPTState =
*PSPT.getStateAt(Iter == BB.begin() ? (ProgramPoint)&BB : &*(--Iter));
I = processInsertionsList(I, &BB, List, SPTState.first, SPTState.second);
auto IterOrErr =
processInsertionsList(I, &BB, List, SPTState.first, SPTState.second);
if (auto E = IterOrErr.takeError())
return Error(std::move(E));
I = *IterOrErr;
}
// Process insertions at the end of bb
auto WRI = Todo.find(&BB);
if (WRI != Todo.end()) {
std::pair<int, int> SPTState = *PSPT.getStateAt(*BB.rbegin());
processInsertionsList(BB.end(), &BB, WRI->second, SPTState.first,
SPTState.second);
if (auto E = processInsertionsList(BB.end(), &BB, WRI->second,
SPTState.first, SPTState.second)
.takeError())
return Error(std::move(E));
Changes = true;
}
}
Expand Down Expand Up @@ -1945,7 +1958,7 @@ void ShrinkWrapping::rebuildCFI() {
}
}

bool ShrinkWrapping::perform(bool HotOnly) {
Expected<bool> ShrinkWrapping::perform(bool HotOnly) {
HasDeletedOffsetCFIs = BitVector(BC.MRI->getNumRegs(), false);
PushOffsetByReg = std::vector<int64_t>(BC.MRI->getNumRegs(), 0LL);
PopOffsetByReg = std::vector<int64_t>(BC.MRI->getNumRegs(), 0LL);
Expand Down Expand Up @@ -1998,7 +2011,11 @@ bool ShrinkWrapping::perform(bool HotOnly) {
});
SLM.performChanges();
// Early exit if processInsertions doesn't detect any todo items
if (!processInsertions())
auto ModifiedOrErr = processInsertions();
if (auto E = ModifiedOrErr.takeError())
return Error(std::move(E));
const bool Modified = *ModifiedOrErr;
if (!Modified)
return false;
processDeletions();
if (foldIdenticalSplitEdges()) {
Expand All @@ -2018,28 +2035,28 @@ bool ShrinkWrapping::perform(bool HotOnly) {
return true;
}

void ShrinkWrapping::printStats() {
outs() << "BOLT-INFO: Shrink wrapping moved " << SpillsMovedRegularMode
<< " spills inserting load/stores and " << SpillsMovedPushPopMode
<< " spills inserting push/pops\n";
void ShrinkWrapping::printStats(BinaryContext &BC) {
BC.outs() << "BOLT-INFO: Shrink wrapping moved " << SpillsMovedRegularMode
<< " spills inserting load/stores and " << SpillsMovedPushPopMode
<< " spills inserting push/pops\n";
if (!InstrDynamicCount || !StoreDynamicCount)
return;
outs() << "BOLT-INFO: Shrink wrapping reduced " << SpillsMovedDynamicCount
<< " store executions ("
<< format("%.1lf%%",
(100.0 * SpillsMovedDynamicCount / InstrDynamicCount))
<< " total instructions executed, "
<< format("%.1lf%%",
(100.0 * SpillsMovedDynamicCount / StoreDynamicCount))
<< " store instructions)\n";
outs() << "BOLT-INFO: Shrink wrapping failed at reducing "
<< SpillsFailedDynamicCount << " store executions ("
<< format("%.1lf%%",
(100.0 * SpillsFailedDynamicCount / InstrDynamicCount))
<< " total instructions executed, "
<< format("%.1lf%%",
(100.0 * SpillsFailedDynamicCount / StoreDynamicCount))
<< " store instructions)\n";
BC.outs() << "BOLT-INFO: Shrink wrapping reduced " << SpillsMovedDynamicCount
<< " store executions ("
<< format("%.1lf%%",
(100.0 * SpillsMovedDynamicCount / InstrDynamicCount))
<< " total instructions executed, "
<< format("%.1lf%%",
(100.0 * SpillsMovedDynamicCount / StoreDynamicCount))
<< " store instructions)\n";
BC.outs() << "BOLT-INFO: Shrink wrapping failed at reducing "
<< SpillsFailedDynamicCount << " store executions ("
<< format("%.1lf%%",
(100.0 * SpillsFailedDynamicCount / InstrDynamicCount))
<< " total instructions executed, "
<< format("%.1lf%%",
(100.0 * SpillsFailedDynamicCount / StoreDynamicCount))
<< " store instructions)\n";
}

// Operators necessary as a result of using MCAnnotation
Expand Down
22 changes: 12 additions & 10 deletions bolt/lib/Passes/SplitFunctions.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -712,15 +712,15 @@ bool SplitFunctions::shouldOptimize(const BinaryFunction &BF) const {
return BinaryFunctionPass::shouldOptimize(BF);
}

void SplitFunctions::runOnFunctions(BinaryContext &BC) {
Error SplitFunctions::runOnFunctions(BinaryContext &BC) {
if (!opts::SplitFunctions)
return;
return Error::success();

// If split strategy is not CDSplit, then a second run of the pass is not
// needed after function reordering.
if (BC.HasFinalizedFunctionOrder &&
opts::SplitStrategy != SplitFunctionsStrategy::CDSplit)
return;
return Error::success();

std::unique_ptr<SplitStrategy> Strategy;
bool ForceSequential = false;
Expand Down Expand Up @@ -766,10 +766,12 @@ void SplitFunctions::runOnFunctions(BinaryContext &BC) {
"SplitFunctions", ForceSequential);

if (SplitBytesHot + SplitBytesCold > 0)
outs() << "BOLT-INFO: splitting separates " << SplitBytesHot
<< " hot bytes from " << SplitBytesCold << " cold bytes "
<< format("(%.2lf%% of split functions is hot).\n",
100.0 * SplitBytesHot / (SplitBytesHot + SplitBytesCold));
BC.outs() << "BOLT-INFO: splitting separates " << SplitBytesHot
<< " hot bytes from " << SplitBytesCold << " cold bytes "
<< format("(%.2lf%% of split functions is hot).\n",
100.0 * SplitBytesHot /
(SplitBytesHot + SplitBytesCold));
return Error::success();
}

void SplitFunctions::splitFunction(BinaryFunction &BF, SplitStrategy &S) {
Expand Down Expand Up @@ -899,9 +901,9 @@ void SplitFunctions::splitFunction(BinaryFunction &BF, SplitStrategy &S) {
if (alignTo(OriginalHotSize, opts::SplitAlignThreshold) <=
alignTo(HotSize, opts::SplitAlignThreshold) + opts::SplitThreshold) {
if (opts::Verbosity >= 2) {
outs() << "BOLT-INFO: Reversing splitting of function "
<< formatv("{0}:\n {1:x}, {2:x} -> {3:x}\n", BF, HotSize,
ColdSize, OriginalHotSize);
BC.outs() << "BOLT-INFO: Reversing splitting of function "
<< formatv("{0}:\n {1:x}, {2:x} -> {3:x}\n", BF, HotSize,
ColdSize, OriginalHotSize);
}

// Reverse the action of createEHTrampolines(). The trampolines will be
Expand Down
16 changes: 9 additions & 7 deletions bolt/lib/Passes/StokeInfo.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,8 @@ bool StokeInfo::checkFunction(BinaryFunction &BF, DataflowInfoManager &DInfo,

if (!BF.isSimple() || BF.isMultiEntry() || BF.empty())
return false;
outs() << " STOKE-INFO: analyzing function " << Name << "\n";
BF.getBinaryContext().outs()
<< " STOKE-INFO: analyzing function " << Name << "\n";

FuncInfo.FuncName = Name;
FuncInfo.Offset = BF.getFileOffset();
Expand Down Expand Up @@ -140,19 +141,19 @@ bool StokeInfo::checkFunction(BinaryFunction &BF, DataflowInfoManager &DInfo,
LiveOutBV &= DefaultLiveOutMask;
getRegNameFromBitVec(BF.getBinaryContext(), LiveOutBV, &FuncInfo.LiveOut);

outs() << " STOKE-INFO: end function \n";
BF.getBinaryContext().outs() << " STOKE-INFO: end function \n";
return true;
}

void StokeInfo::runOnFunctions(BinaryContext &BC) {
outs() << "STOKE-INFO: begin of stoke pass\n";
Error StokeInfo::runOnFunctions(BinaryContext &BC) {
BC.outs() << "STOKE-INFO: begin of stoke pass\n";

std::ofstream Outfile;
if (!opts::StokeOutputDataFilename.empty()) {
Outfile.open(opts::StokeOutputDataFilename);
} else {
errs() << "STOKE-INFO: output file is required\n";
return;
BC.errs() << "STOKE-INFO: output file is required\n";
return Error::success();
}

// check some context meta data
Expand Down Expand Up @@ -185,7 +186,8 @@ void StokeInfo::runOnFunctions(BinaryContext &BC) {
FuncInfo.printData(Outfile);
}

outs() << "STOKE-INFO: end of stoke pass\n";
BC.outs() << "STOKE-INFO: end of stoke pass\n";
return Error::success();
}

} // namespace bolt
Expand Down
31 changes: 17 additions & 14 deletions bolt/lib/Passes/TailDuplication.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -633,9 +633,9 @@ void TailDuplication::runOnFunction(BinaryFunction &Function) {
ModifiedFunctions++;
}

void TailDuplication::runOnFunctions(BinaryContext &BC) {
Error TailDuplication::runOnFunctions(BinaryContext &BC) {
if (opts::TailDuplicationMode == TailDuplication::TD_NONE)
return;
return Error::success();

for (auto &It : BC.getBinaryFunctions()) {
BinaryFunction &Function = It.second;
Expand All @@ -644,23 +644,26 @@ void TailDuplication::runOnFunctions(BinaryContext &BC) {
runOnFunction(Function);
}

outs() << "BOLT-INFO: tail duplication"
<< format(" modified %zu (%.2f%%) functions;", ModifiedFunctions,
100.0 * ModifiedFunctions / BC.getBinaryFunctions().size())
<< format(" duplicated %zu blocks (%zu bytes) responsible for",
DuplicatedBlockCount, DuplicatedByteCount)
<< format(" %zu dynamic executions (%.2f%% of all block executions)",
DuplicationsDynamicCount,
100.0 * DuplicationsDynamicCount / AllDynamicCount)
<< "\n";
BC.outs()
<< "BOLT-INFO: tail duplication"
<< format(" modified %zu (%.2f%%) functions;", ModifiedFunctions,
100.0 * ModifiedFunctions / BC.getBinaryFunctions().size())
<< format(" duplicated %zu blocks (%zu bytes) responsible for",
DuplicatedBlockCount, DuplicatedByteCount)
<< format(" %zu dynamic executions (%.2f%% of all block executions)",
DuplicationsDynamicCount,
100.0 * DuplicationsDynamicCount / AllDynamicCount)
<< "\n";

if (opts::TailDuplicationConstCopyPropagation) {
outs() << "BOLT-INFO: tail duplication "
<< format("applied %zu static and %zu dynamic propagation deletions",
BC.outs() << "BOLT-INFO: tail duplication "
<< format(
"applied %zu static and %zu dynamic propagation deletions",
StaticInstructionDeletionCount,
DynamicInstructionDeletionCount)
<< "\n";
<< "\n";
}
return Error::success();
}

} // end namespace bolt
Expand Down
7 changes: 4 additions & 3 deletions bolt/lib/Passes/ThreeWayBranch.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -147,16 +147,17 @@ void ThreeWayBranch::runOnFunction(BinaryFunction &Function) {
}
}

void ThreeWayBranch::runOnFunctions(BinaryContext &BC) {
Error ThreeWayBranch::runOnFunctions(BinaryContext &BC) {
for (auto &It : BC.getBinaryFunctions()) {
BinaryFunction &Function = It.second;
if (!shouldRunOnFunction(Function))
continue;
runOnFunction(Function);
}

outs() << "BOLT-INFO: number of three way branches order changed: "
<< BranchesAltered << "\n";
BC.outs() << "BOLT-INFO: number of three way branches order changed: "
<< BranchesAltered << "\n";
return Error::success();
}

} // end namespace bolt
Expand Down
14 changes: 8 additions & 6 deletions bolt/lib/Passes/ValidateInternalCalls.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -302,9 +302,9 @@ bool ValidateInternalCalls::analyzeFunction(BinaryFunction &Function) const {
return true;
}

void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
Error ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
if (!BC.isX86())
return;
return Error::success();

// Look for functions that need validation. This should be pretty rare.
std::set<BinaryFunction *> NeedsValidation;
Expand All @@ -323,7 +323,7 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {

// Skip validation for non-relocation mode
if (!BC.HasRelocations)
return;
return Error::success();

// Since few functions need validation, we can work with our most expensive
// algorithms here. Fix the CFG treating internal calls as unconditional
Expand All @@ -339,13 +339,15 @@ void ValidateInternalCalls::runOnFunctions(BinaryContext &BC) {
}

if (!Invalid.empty()) {
errs() << "BOLT-WARNING: will skip the following function(s) as unsupported"
" internal calls were detected:\n";
BC.errs()
<< "BOLT-WARNING: will skip the following function(s) as unsupported"
" internal calls were detected:\n";
for (BinaryFunction *Function : Invalid) {
errs() << " " << *Function << "\n";
BC.errs() << " " << *Function << "\n";
Function->setIgnored();
}
}
return Error::success();
}

} // namespace bolt
Expand Down
13 changes: 7 additions & 6 deletions bolt/lib/Passes/ValidateMemRefs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,13 @@ void ValidateMemRefs::runOnFunction(BinaryFunction &BF) {
}
}

void ValidateMemRefs::runOnFunctions(BinaryContext &BC) {
Error ValidateMemRefs::runOnFunctions(BinaryContext &BC) {
if (!BC.isX86())
return;
return Error::success();

// Skip validation if not moving JT
if (opts::JumpTables == JTS_NONE || opts::JumpTables == JTS_BASIC)
return;
return Error::success();

ParallelUtilities::WorkFuncWithAllocTy ProcessFunction =
[&](BinaryFunction &BF, MCPlusBuilder::AllocatorIdTy AllocId) {
Expand All @@ -94,10 +94,11 @@ void ValidateMemRefs::runOnFunctions(BinaryContext &BC) {
LLVM_DEBUG(dbgs() << "BOLT-DEBUG: memrefs validation is concluded\n");

if (!ReplacedReferences)
return;
return Error::success();

outs() << "BOLT-INFO: validate-mem-refs updated " << ReplacedReferences
<< " object references\n";
BC.outs() << "BOLT-INFO: validate-mem-refs updated " << ReplacedReferences
<< " object references\n";
return Error::success();
}

} // namespace llvm::bolt
13 changes: 7 additions & 6 deletions bolt/lib/Passes/VeneerElimination.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ static llvm::cl::opt<bool>
namespace llvm {
namespace bolt {

void VeneerElimination::runOnFunctions(BinaryContext &BC) {
Error VeneerElimination::runOnFunctions(BinaryContext &BC) {
if (!opts::EliminateVeneers || !BC.isAArch64())
return;
return Error::success();

std::map<uint64_t, BinaryFunction> &BFs = BC.getBinaryFunctions();
std::unordered_map<const MCSymbol *, const MCSymbol *> VeneerDestinations;
Expand All @@ -51,8 +51,8 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) {
VeneerDestinations[Symbol] = VeneerTargetSymbol;
}

outs() << "BOLT-INFO: number of removed linker-inserted veneers: "
<< VeneersCount << "\n";
BC.outs() << "BOLT-INFO: number of removed linker-inserted veneers: "
<< VeneersCount << "\n";

// Handle veneers to veneers in case they occur
for (auto &Entry : VeneerDestinations) {
Expand All @@ -79,8 +79,8 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) {
VeneerCallers++;
if (!BC.MIB->replaceBranchTarget(
Instr, VeneerDestinations[TargetSymbol], BC.Ctx.get())) {
errs() << "BOLT-ERROR: updating veneer call destination failed\n";
exit(1);
return createFatalBOLTError(
"BOLT-ERROR: updating veneer call destination failed\n");
}
}
}
Expand All @@ -90,6 +90,7 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) {
dbgs() << "BOLT-INFO: number of linker-inserted veneers call sites: "
<< VeneerCallers << "\n");
(void)VeneerCallers;
return Error::success();
}

} // namespace bolt
Expand Down
6 changes: 3 additions & 3 deletions bolt/lib/Profile/BoltAddressTranslation.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ void BoltAddressTranslation::write(const BinaryContext &BC, raw_ostream &OS) {
writeMaps</*Cold=*/false>(Maps, PrevAddress, OS);
writeMaps</*Cold=*/true>(Maps, PrevAddress, OS);

outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n";
BC.outs() << "BOLT-INFO: Wrote " << Maps.size() << " BAT maps\n";
}

APInt BoltAddressTranslation::calculateBranchEntriesBitMask(MapTy &Map,
Expand Down Expand Up @@ -201,7 +201,7 @@ void BoltAddressTranslation::writeMaps(std::map<uint64_t, MapTy> &Maps,
}
}

std::error_code BoltAddressTranslation::parse(StringRef Buf) {
std::error_code BoltAddressTranslation::parse(raw_ostream &OS, StringRef Buf) {
DataExtractor DE = DataExtractor(Buf, true, 8);
uint64_t Offset = 0;
if (Buf.size() < 12)
Expand All @@ -225,7 +225,7 @@ std::error_code BoltAddressTranslation::parse(StringRef Buf) {
uint64_t PrevAddress = 0;
parseMaps</*Cold=*/false>(HotFuncs, PrevAddress, DE, Offset, Err);
parseMaps</*Cold=*/true>(HotFuncs, PrevAddress, DE, Offset, Err);
outs() << "BOLT-INFO: Parsed " << Maps.size() << " BAT entries\n";
OS << "BOLT-INFO: Parsed " << Maps.size() << " BAT entries\n";
return errorToErrorCode(std::move(Err));
}

Expand Down
30 changes: 19 additions & 11 deletions bolt/lib/Rewrite/BinaryPassManager.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ const char BinaryFunctionPassManager::TimerGroupName[] = "passman";
const char BinaryFunctionPassManager::TimerGroupDesc[] =
"Binary Function Pass Manager";

void BinaryFunctionPassManager::runPasses() {
Error BinaryFunctionPassManager::runPasses() {
auto &BFs = BC.getBinaryFunctions();
for (size_t PassIdx = 0; PassIdx < Passes.size(); PassIdx++) {
const std::pair<const bool, std::unique_ptr<BinaryFunctionPass>>
Expand All @@ -281,13 +281,20 @@ void BinaryFunctionPassManager::runPasses() {
formatv("{0:2}_{1}", PassIdx, Pass->getName()).str();

if (opts::Verbosity > 0)
outs() << "BOLT-INFO: Starting pass: " << Pass->getName() << "\n";
BC.outs() << "BOLT-INFO: Starting pass: " << Pass->getName() << "\n";

NamedRegionTimer T(Pass->getName(), Pass->getName(), TimerGroupName,
TimerGroupDesc, TimeOpts);

callWithDynoStats([this, &Pass] { Pass->runOnFunctions(BC); }, BFs,
Pass->getName(), opts::DynoStatsAll, BC.isAArch64());
Error E = Error::success();
callWithDynoStats(
BC.outs(),
[this, &E, &Pass] {
E = joinErrors(std::move(E), Pass->runOnFunctions(BC));
},
BFs, Pass->getName(), opts::DynoStatsAll, BC.isAArch64());
if (E)
return Error(std::move(E));

if (opts::VerifyCFG &&
!std::accumulate(
Expand All @@ -296,13 +303,13 @@ void BinaryFunctionPassManager::runPasses() {
const std::pair<const uint64_t, BinaryFunction> &It) {
return Valid && It.second.validateCFG();
})) {
errs() << "BOLT-ERROR: Invalid CFG detected after pass "
<< Pass->getName() << "\n";
exit(1);
return createFatalBOLTError(
Twine("BOLT-ERROR: Invalid CFG detected after pass ") +
Twine(Pass->getName()) + Twine("\n"));
}

if (opts::Verbosity > 0)
outs() << "BOLT-INFO: Finished pass: " << Pass->getName() << "\n";
BC.outs() << "BOLT-INFO: Finished pass: " << Pass->getName() << "\n";

if (!opts::PrintAll && !opts::DumpDotAll && !Pass->printPass())
continue;
Expand All @@ -315,15 +322,16 @@ void BinaryFunctionPassManager::runPasses() {
if (!Pass->shouldPrint(Function))
continue;

Function.print(outs(), Message);
Function.print(BC.outs(), Message);

if (opts::DumpDotAll)
Function.dumpGraphForPass(PassIdName);
}
}
return Error::success();
}

void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
Error BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
BinaryFunctionPassManager Manager(BC);

const DynoStats InitialDynoStats =
Expand Down Expand Up @@ -516,7 +524,7 @@ void BinaryFunctionPassManager::runAllPasses(BinaryContext &BC) {
// in parallel and restore them
Manager.registerPass(std::make_unique<CleanMCState>(NeverPrint));

Manager.runPasses();
return Manager.runPasses();
}

} // namespace bolt
Expand Down
8 changes: 4 additions & 4 deletions bolt/lib/Rewrite/BoltDiff.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -294,9 +294,9 @@ class RewriteInstanceDiff {
}
PrintProgramStats PPS(opts::NeverPrint);
outs() << "* BOLT-DIFF: Starting print program stats pass for binary 1\n";
PPS.runOnFunctions(*RI1.BC);
RI1.BC->logBOLTErrorsAndQuitOnFatal(PPS.runOnFunctions(*RI1.BC));
outs() << "* BOLT-DIFF: Starting print program stats pass for binary 2\n";
PPS.runOnFunctions(*RI2.BC);
RI1.BC->logBOLTErrorsAndQuitOnFatal(PPS.runOnFunctions(*RI2.BC));
outs() << "=====\n";
outs() << "Inputs share " << BothHaveProfile
<< " functions with valid profile.\n";
Expand Down Expand Up @@ -700,9 +700,9 @@ void RewriteInstance::compare(RewriteInstance &RI2) {
if (opts::ICF) {
IdenticalCodeFolding ICF(opts::NeverPrint);
outs() << "BOLT-DIFF: Starting ICF pass for binary 1";
ICF.runOnFunctions(*BC);
BC->logBOLTErrorsAndQuitOnFatal(ICF.runOnFunctions(*BC));
outs() << "BOLT-DIFF: Starting ICF pass for binary 2";
ICF.runOnFunctions(*RI2.BC);
BC->logBOLTErrorsAndQuitOnFatal(ICF.runOnFunctions(*RI2.BC));
}

RewriteInstanceDiff RID(*this, RI2);
Expand Down
7 changes: 4 additions & 3 deletions bolt/lib/Rewrite/DWARFRewriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -709,7 +709,7 @@ void DWARFRewriter::updateDebugInfo() {
: LegacyRangesSectionWriter.get();
// Skipping CUs that failed to load.
if (SplitCU) {
DIEBuilder DWODIEBuilder(&(*SplitCU)->getContext(), true);
DIEBuilder DWODIEBuilder(BC, &(*SplitCU)->getContext(), true);
DWODIEBuilder.buildDWOUnit(**SplitCU);
std::string DWOName = updateDWONameCompDir(
*Unit, *DIEBlder, *DIEBlder->getUnitDIEbyUnit(*Unit));
Expand Down Expand Up @@ -754,7 +754,7 @@ void DWARFRewriter::updateDebugInfo() {
AddrWriter->update(*DIEBlder, *Unit);
};

DIEBuilder DIEBlder(BC.DwCtx.get());
DIEBuilder DIEBlder(BC, BC.DwCtx.get());
DIEBlder.buildTypeUnits(StrOffstsWriter.get());
SmallVector<char, 20> OutBuffer;
std::unique_ptr<raw_svector_ostream> ObjOS =
Expand Down Expand Up @@ -1655,7 +1655,8 @@ createDwarfOnlyBC(const object::ObjectFile &File) {
&File, false,
DWARFContext::create(File, DWARFContext::ProcessDebugRelocations::Ignore,
nullptr, "", WithColor::defaultErrorHandler,
WithColor::defaultWarningHandler)));
WithColor::defaultWarningHandler),
{llvm::outs(), llvm::errs()}));
}

StringMap<DWARFRewriter::KnownSectionsEntry>
Expand Down
12 changes: 5 additions & 7 deletions bolt/lib/Rewrite/MachORewriteInstance.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,8 @@ MachORewriteInstance::MachORewriteInstance(object::MachOObjectFile *InputFile,
: InputFile(InputFile), ToolPath(ToolPath) {
ErrorAsOutParameter EAO(&Err);
auto BCOrErr = BinaryContext::createBinaryContext(
InputFile, /* IsPIC */ true, DWARFContext::create(*InputFile));
InputFile, /* IsPIC */ true, DWARFContext::create(*InputFile),
{llvm::outs(), llvm::errs()});
if (Error E = BCOrErr.takeError()) {
Err = std::move(E);
return;
Expand Down Expand Up @@ -337,7 +338,7 @@ void MachORewriteInstance::disassembleFunctions() {
BinaryFunction &Function = BFI.second;
if (!Function.isSimple())
continue;
Function.disassemble();
BC->logBOLTErrorsAndQuitOnFatal(Function.disassemble());
if (opts::PrintDisasm)
Function.print(outs(), "after disassembly");
}
Expand All @@ -348,10 +349,7 @@ void MachORewriteInstance::buildFunctionsCFG() {
BinaryFunction &Function = BFI.second;
if (!Function.isSimple())
continue;
if (!Function.buildCFG(/*AllocId*/ 0)) {
errs() << "BOLT-WARNING: failed to build CFG for the function "
<< Function << "\n";
}
BC->logBOLTErrorsAndQuitOnFatal(Function.buildCFG(/*AllocId*/ 0));
}
}

Expand Down Expand Up @@ -387,7 +385,7 @@ void MachORewriteInstance::runOptimizationPasses() {
Manager.registerPass(
std::make_unique<FinalizeFunctions>(opts::PrintFinalized));

Manager.runPasses();
BC->logBOLTErrorsAndQuitOnFatal(Manager.runPasses());
}

void MachORewriteInstance::mapInstrumentationSection(
Expand Down
571 changes: 319 additions & 252 deletions bolt/lib/Rewrite/RewriteInstance.cpp

Large diffs are not rendered by default.

18 changes: 13 additions & 5 deletions bolt/lib/Target/X86/X86MCSymbolizer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,13 @@ bool X86MCSymbolizer::tryAddingSymbolicOperand(
// a PC-relative 8-byte fixup, which is what we need to cover this. The
// only way to do this is to use the symbol name _GLOBAL_OFFSET_TABLE_.
if (Relocation::isX86GOTPC64(Relocation->Type)) {
auto [Sym, Addend] = handleGOTPC64(*Relocation, InstAddress);
auto PairOrErr = handleGOTPC64(*Relocation, InstAddress);
if (auto E = PairOrErr.takeError()) {
Function.setSimple(false);
BC.logBOLTErrorsAndQuitOnFatal(std::move(E));
return false;
}
auto [Sym, Addend] = *PairOrErr;
addOperand(Sym, Addend);
return true;
}
Expand All @@ -158,14 +164,16 @@ bool X86MCSymbolizer::tryAddingSymbolicOperand(
return true;
}

std::pair<MCSymbol *, uint64_t>
Expected<std::pair<MCSymbol *, uint64_t>>
X86MCSymbolizer::handleGOTPC64(const Relocation &R, uint64_t InstrAddr) {
BinaryContext &BC = Function.getBinaryContext();
const BinaryData *GOTSymBD = BC.getGOTSymbol();
if (!GOTSymBD || !GOTSymBD->getAddress()) {
errs() << "BOLT-ERROR: R_X86_GOTPC64 relocation is present but we did "
"not detect a valid _GLOBAL_OFFSET_TABLE_ in symbol table\n";
exit(1);
// This error is pretty serious but we can't kill the disassembler
// because of it, so don't make it fatal. Log it and warn the user.
return createNonFatalBOLTError(
"R_X86_GOTPC64 relocation is present but we did not detect "
"a valid _GLOBAL_OFFSET_TABLE_ in symbol table\n");
}
// R_X86_GOTPC64 are not relative to the Reloc nor end of instruction,
// but the start of the MOVABSQ instruction. So the Target Address is
Expand Down
4 changes: 2 additions & 2 deletions bolt/lib/Target/X86/X86MCSymbolizer.h
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ class X86MCSymbolizer : public MCSymbolizer {
BinaryFunction &Function;
bool CreateNewSymbols{true};

std::pair<MCSymbol *, uint64_t> handleGOTPC64(const Relocation &R,
uint64_t InstrAddr);
Expected<std::pair<MCSymbol *, uint64_t>> handleGOTPC64(const Relocation &R,
uint64_t InstrAddr);

public:
X86MCSymbolizer(BinaryFunction &Function, bool CreateNewSymbols = true)
Expand Down
39 changes: 39 additions & 0 deletions bolt/test/X86/fatal-error.s
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Tests whether llvm-bolt will correctly exit with error code and printing
# fatal error message in case one occurs. Here we test opening a function
# reordering file that does not exist.

# RUN: llvm-mc -filetype=obj -triple x86_64-unknown-unknown %s -o %t.o
# RUN: %clang %cflags %t.o -o %t.exe -Wl,-q
# RUN: not llvm-bolt %t.exe -o %t.null \
# RUN: --reorder-blocks=normal --reorder-functions=user \
# RUN: --function-order=/DOES/NOT/EXIST 2>&1 \
# RUN: | FileCheck --check-prefix=CHECK %s

# CHECK: FATAL BOLT-ERROR: Ordered functions file "/DOES/NOT/EXIST" can't be opened

# Sample function reordering input, based off function-order-lite.s
.globl main
.type main, %function
main:
.cfi_startproc
.LBB06:
callq func_a
retq
.cfi_endproc
.size main, .-main

.globl func_a
.type func_a, %function
func_a:
.cfi_startproc
retq
.cfi_endproc
.size func_a, .-func_a

.globl func_b
.type func_b, %function
func_b:
.cfi_startproc
retq
.cfi_endproc
.size func_b, .-func_b
19 changes: 19 additions & 0 deletions bolt/test/X86/log.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Tests whether llvm-bolt is able to redirect logs when processing a simple
# input. If this test fails on your changes, please use BinaryContext::outs()
# to print BOLT logging instead of llvm::outs().

RUN: yaml2obj %p/Inputs/blarge.yaml &> %t.exe
RUN: llvm-bolt %t.exe -o %t.null --data %p/Inputs/blarge.fdata -v=2 \
RUN: --reorder-blocks=normal --print-finalized --log-file=%t.log 2>&1 \
RUN: | FileCheck --check-prefix=CHECK --allow-empty %s
RUN: cat %t.log | FileCheck %s --check-prefix=CHECK-LOG

CHECK-NOT: BOLT-INFO
CHECK-NOT: BOLT-WARNING
CHECK-NOT: BOLT-ERROR

# Check some usual BOLT output lines are being redirected to the log file
CHECK-LOG: BOLT-INFO: Target architecture
CHECK-LOG: BOLT-INFO: BOLT version
CHECK-LOG: BOLT-INFO: basic block reordering modified layout
CHECK-LOG: Binary Function "usqrt"
2 changes: 1 addition & 1 deletion bolt/tools/bat-dump/bat-dump.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ void dumpBATFor(llvm::object::ELFObjectFileBase *InputFile) {
exit(1);
}

if (std::error_code EC = BAT.parse(SectionContents)) {
if (std::error_code EC = BAT.parse(outs(), SectionContents)) {
errs() << "BOLT-ERROR: failed to parse BOLT address translation "
"table. Malformed BAT section\n";
exit(1);
Expand Down
26 changes: 25 additions & 1 deletion bolt/tools/driver/llvm-bolt.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ BoltProfile("b",
cl::aliasopt(InputDataFilename),
cl::cat(BoltCategory));

cl::opt<std::string>
LogFile("log-file",
cl::desc("redirect journaling to a file instead of stdout/stderr"),
cl::Hidden, cl::cat(BoltCategory));

static cl::opt<std::string>
InputDataFilename2("data2",
cl::desc("<data file>"),
Expand Down Expand Up @@ -207,6 +212,24 @@ int main(int argc, char **argv) {
if (!sys::fs::exists(opts::InputFilename))
report_error(opts::InputFilename, errc::no_such_file_or_directory);

// Initialize journaling streams
raw_ostream *BOLTJournalOut = &outs();
raw_ostream *BOLTJournalErr = &errs();
// RAII obj to keep log file open throughout execution
std::unique_ptr<raw_fd_ostream> LogFileStream;
if (!opts::LogFile.empty()) {
std::error_code LogEC;
LogFileStream = std::make_unique<raw_fd_ostream>(
opts::LogFile, LogEC, sys::fs::OpenFlags::OF_None);
if (LogEC) {
errs() << "BOLT-ERROR: cannot open requested log file for writing: "
<< LogEC.message() << "\n";
exit(1);
}
BOLTJournalOut = LogFileStream.get();
BOLTJournalErr = LogFileStream.get();
}

// Attempt to open the binary.
if (!opts::DiffOnly) {
Expected<OwningBinary<Binary>> BinaryOrErr =
Expand All @@ -216,7 +239,8 @@ int main(int argc, char **argv) {
Binary &Binary = *BinaryOrErr.get().getBinary();

if (auto *e = dyn_cast<ELFObjectFileBase>(&Binary)) {
auto RIOrErr = RewriteInstance::create(e, argc, argv, ToolPath);
auto RIOrErr = RewriteInstance::create(e, argc, argv, ToolPath,
*BOLTJournalOut, *BOLTJournalErr);
if (Error E = RIOrErr.takeError())
report_error(opts::InputFilename, std::move(E));
RewriteInstance &RI = *RIOrErr.get();
Expand Down
3 changes: 2 additions & 1 deletion bolt/unittests/Core/BinaryContext.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ struct BinaryContextTester : public testing::TestWithParam<Triple::ArchType> {

void initializeBOLT() {
BC = cantFail(BinaryContext::createBinaryContext(
ObjFile.get(), true, DWARFContext::create(*ObjFile.get())));
ObjFile.get(), true, DWARFContext::create(*ObjFile.get()),
{llvm::outs(), llvm::errs()}));
ASSERT_FALSE(!BC);
}

Expand Down
3 changes: 2 additions & 1 deletion bolt/unittests/Core/MCPlusBuilder.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ struct MCPlusBuilderTester : public testing::TestWithParam<Triple::ArchType> {

void initializeBolt() {
BC = cantFail(BinaryContext::createBinaryContext(
ObjFile.get(), true, DWARFContext::create(*ObjFile.get())));
ObjFile.get(), true, DWARFContext::create(*ObjFile.get()),
{llvm::outs(), llvm::errs()}));
ASSERT_FALSE(!BC);
BC->initializeTarget(std::unique_ptr<MCPlusBuilder>(
createMCPlusBuilder(GetParam(), BC->MIA.get(), BC->MII.get(),
Expand Down